File indexing completed on 2022-11-23 10:26:22

0001 #!/usr/bin/env python3
0002 #
0003 # SPDX-FileCopyrightText: 2018-2020 Aleix Pol Gonzalez <aleixpol@kde.org>
0004 # SPDX-FileCopyrightText: 2019-2020 Ben Cooksley <bcooksley@kde.org>
0005 # SPDX-FileCopyrightText: 2020 Volker Krause <vkrause@kde.org>
0006 #
0007 # SPDX-License-Identifier: GPL-2.0-or-later
0008 #
0009 # Generates fastlane metadata for Android apps from appstream files.
0010 #
0011 
0012 import argparse
0013 import glob
0014 import io
0015 import os
0016 import re
0017 import requests
0018 import shutil
0019 import subprocess
0020 import sys
0021 import tempfile
0022 import xdg.DesktopEntry
0023 import xml.etree.ElementTree as ET
0024 import yaml
0025 import zipfile
0026 
0027 # Constants used in this script
0028 # map KDE's translated language codes to those expected by Android
0029 # see https://f-droid.org/en/docs/Translation_and_Localization/
0030 # F-Droid is more tolerant than the Play Store here, the latter rejects anything not exactly matching its known codes
0031 # Android does do the expected fallbacks, so the seemingly "too specific" mappings here are still working as expected
0032 # see https://developer.android.com/guide/topics/resources/multilingual-support#resource-resolution-examples
0033 languageMap = {
0034     None: "en-US",
0035     "ca-valencia": None, # not supported by Android
0036     "cs": "cs-CZ",
0037     "de": "de-DE",
0038     "es": "es-ES",
0039     "eu": "eu-ES",
0040     "fi": "fi-FI",
0041     "fr": "fr-FR",
0042     "gl": "gl-ES",
0043     "ia": None, # not supported by Android
0044     "it": "it-IT",
0045     "ko": "ko-KR",
0046     "nl": "nl-NL",
0047     "pl": "pl-PL",
0048     "pt": "pt-PT",
0049     "ru": "ru-RU",
0050     "sr": "sr-Cyrl-RS",
0051     "sr@latin": "sr-Latn-RS",
0052     "sv": "sv-SE",
0053     'x-test': None
0054 }
0055 
0056 # The subset of supported rich text tags in F-Droid and Google Play
0057 # - see https://f-droid.org/en/docs/All_About_Descriptions_Graphics_and_Screenshots/ for F-Droid
0058 # - Google Play doesn't support lists
0059 supportedRichTextTags = { 'b', 'u', 'i' }
0060 
0061 # List all translated languages present in an Appstream XML file
0062 def listAllLanguages(root, langs):
0063     for elem in root:
0064         lang = elem.get('{http://www.w3.org/XML/1998/namespace}lang')
0065         if not lang in langs:
0066             langs.add(lang)
0067         listAllLanguages(elem, langs)
0068 
0069 # Apply language fallback to a map of translations
0070 def applyLanguageFallback(data, allLanguages):
0071     for l in allLanguages:
0072         if not l in data or not data[l] or len(data[l]) == 0:
0073             data[l] = data[None]
0074 
0075 # Android appdata.xml textual item parser
0076 # This function handles reading standard text entries within an Android appdata.xml file
0077 # In particular, it handles splitting out the various translations, and converts some HTML to something which F-Droid can make use of
0078 # We have to handle incomplete translations both on top-level and intermediate tags,
0079 # and fall back to the English default text where necessary.
0080 def readText(elem, found, allLanguages):
0081     # Determine the language this entry is in
0082     lang = elem.get('{http://www.w3.org/XML/1998/namespace}lang')
0083 
0084     # Do we have any text for this language yet?
0085     # If not, get everything setup
0086     for l in allLanguages:
0087         if not l in found:
0088             found[l] = ""
0089 
0090     # If there is text available, we'll want to extract it
0091     # Additionally, if this element has any children, make sure we read those as well
0092     if elem.tag in supportedRichTextTags:
0093         if (elem.text and elem.text.strip()) or lang:
0094             found[lang] += '<' + elem.tag + '>'
0095         else:
0096             for l in allLanguages:
0097                 found[l] += '<' + elem.tag + '>'
0098     elif elem.tag == 'li':
0099         found[lang] += 'ยท '
0100 
0101     if elem.text and elem.text.strip():
0102         found[lang] += elem.text
0103 
0104     subOutput = {}
0105     for child in elem:
0106         if not child.get('{http://www.w3.org/XML/1998/namespace}lang') and len(subOutput) > 0:
0107             applyLanguageFallback(subOutput, allLanguages)
0108             for l in allLanguages:
0109                 found[l] += subOutput[l]
0110             subOutput = {}
0111         readText(child, subOutput, allLanguages)
0112     if len(subOutput) > 0:
0113         applyLanguageFallback(subOutput, allLanguages)
0114         for l in allLanguages:
0115             found[l] += subOutput[l]
0116 
0117     if elem.tag in supportedRichTextTags:
0118         if (elem.text and elem.text.strip()) or lang:
0119             found[lang] += '</' + elem.tag + '>'
0120         else:
0121             for l in allLanguages:
0122                 found[l] += '</' + elem.tag + '>'
0123 
0124     # Finally, if this element is a HTML Paragraph (p) or HTML List Item (li) make sure we add a new line for presentation purposes
0125     if elem.tag == 'li' or elem.tag == 'p':
0126         found[lang] += "\n"
0127 
0128 
0129 # Create the various Fastlane format files per the information we've previously extracted
0130 # These files are laid out following the Fastlane specification (links below)
0131 # https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots
0132 # https://docs.fastlane.tools/actions/supply/
0133 def createFastlaneFile( applicationName, filenameToPopulate, fileContent ):
0134     # Go through each language and content pair we've been given
0135     for lang, text in fileContent.items():
0136         # First, do we need to amend the language id, to turn the Android language ID into something more F-Droid/Fastlane friendly?
0137         languageCode = languageMap.get(lang, lang)
0138         if not languageCode:
0139             continue
0140 
0141         # Next we need to determine the path to the directory we're going to be writing the data into
0142         repositoryBasePath = arguments.output
0143         path = os.path.join( repositoryBasePath, 'metadata',  applicationName, languageCode )
0144 
0145         # Make sure the directory exists
0146         os.makedirs(path, exist_ok=True)
0147 
0148         # Now write out file contents!
0149         with open(path + '/' + filenameToPopulate, 'w') as f:
0150             f.write(text.strip()) # trim whitespaces, to avoid spurious differences after a Google Play roundtrip
0151 
0152 # Create the summary appname.yml file used by F-Droid to summarise this particular entry in the repository
0153 # see https://f-droid.org/en/docs/Build_Metadata_Reference/
0154 def createYml(appname, data):
0155     # Prepare to retrieve the existing information
0156     info = {}
0157 
0158     # Determine the path to the appname.yml file
0159     repositoryBasePath = arguments.output
0160     path = os.path.join( repositoryBasePath, 'metadata', appname + '.yml' )
0161 
0162     # Update the categories first
0163     # Now is also a good time to add 'KDE' to the list of categories as well
0164     if 'categories' in data:
0165         info['Categories'] = data['categories'][None] + ['KDE']
0166     else:
0167         info['Categories']  = ['KDE']
0168 
0169     # Update the general summary as well
0170     info['Summary'] = data['summary'][None]
0171 
0172     # Check to see if we have a Homepage...
0173     if 'url-homepage' in data:
0174         info['WebSite'] = data['url-homepage'][None]
0175 
0176     # What about a bug tracker?
0177     if 'url-bugtracker' in data:
0178         info['IssueTracker'] = data['url-bugtracker'][None]
0179 
0180     if 'project_license' in data:
0181         info["License"] = data['project_license'][None]
0182 
0183     if 'source-repo' in data:
0184         info['SourceCode'] = data['source-repo']
0185 
0186     if 'url-donation' in data:
0187         info['Donate'] = data['url-donation'][None]
0188     else:
0189         info['Donate'] = 'https://kde.org/community/donations/'
0190 
0191     # static data
0192     info['Translation'] = 'https://l10n.kde.org/'
0193 
0194     # Finally, with our updates completed, we can save the updated appname.yml file back to disk
0195     with open(path, 'w') as output:
0196         yaml.dump(info, output, default_flow_style=False)
0197 
0198 # Integrates locally existing image assets into the metadata
0199 def processLocalImages(applicationName, data):
0200     if not os.path.exists(os.path.join(arguments.source, 'fastlane')):
0201         return
0202 
0203     outPath = os.path.abspath(arguments.output);
0204     oldcwd = os.getcwd()
0205     os.chdir(os.path.join(arguments.source, 'fastlane'))
0206 
0207     imageFiles = glob.glob('metadata/**/*.png', recursive=True)
0208     imageFiles.extend(glob.glob('metadata/**/*.jpg', recursive=True))
0209     for image in imageFiles:
0210         # noramlize single- vs multi-app layouts
0211         imageDestName = image.replace('metadata/android', 'metadata/' + applicationName)
0212 
0213         # copy image
0214         os.makedirs(os.path.dirname(os.path.join(outPath, imageDestName)), exist_ok=True)
0215         shutil.copy(image, os.path.join(outPath, imageDestName))
0216 
0217         # if the source already contains screenshots, those override whatever we found in the appstream file
0218         if 'phoneScreenshots' in image:
0219             data['screenshots'] = {}
0220 
0221     os.chdir(oldcwd)
0222 
0223 # Attempt to find the application icon if we haven't gotten that explicitly from processLocalImages
0224 def findIcon(applicationName, iconBaseName):
0225     iconPath = os.path.join(arguments.output, 'metadata', applicationName, 'en-US', 'images', 'icon.png')
0226     if os.path.exists(iconPath):
0227         return
0228 
0229     oldcwd = os.getcwd()
0230     os.chdir(arguments.source)
0231 
0232     iconFiles = glob.glob(f"**/{iconBaseName}-playstore.png", recursive=True)
0233     for icon in iconFiles:
0234         os.makedirs(os.path.dirname(iconPath), exist_ok=True)
0235         shutil.copy(icon, iconPath)
0236         break
0237 
0238     os.chdir(oldcwd)
0239 
0240 # Download screenshots referenced in the appstream data
0241 # see https://f-droid.org/en/docs/All_About_Descriptions_Graphics_and_Screenshots/
0242 def downloadScreenshots(applicationName, data):
0243     if not 'screenshots' in data:
0244         return
0245 
0246     path = os.path.join(arguments.output, 'metadata',  applicationName, 'en-US', 'images', 'phoneScreenshots')
0247     os.makedirs(path, exist_ok=True)
0248 
0249     i = 1 # number screenshots starting at 1 rather than 0 to match what the fastlane tool does
0250     for screenshot in data['screenshots']:
0251         fileName = str(i) + '-' + screenshot[screenshot.rindex('/') + 1:]
0252         r = requests.get(screenshot)
0253         if r.status_code < 400:
0254             with open(os.path.join(path, fileName), 'wb') as f:
0255                 f.write(r.content)
0256             i += 1
0257 
0258 # Put all metadata for the given application name into an archive
0259 # We need this to easily transfer the entire metadata to the signing machine for integration
0260 # into the F-Droid nightly repository
0261 def createMetadataArchive(applicationName):
0262     srcPath = os.path.join(arguments.output, 'metadata')
0263     zipFileName = os.path.join(srcPath, 'fastlane-' + applicationName + '.zip')
0264     if os.path.exists(zipFileName):
0265         os.unlink(zipFileName)
0266     archive = zipfile.ZipFile(zipFileName, 'w')
0267     archive.write(os.path.join(srcPath, applicationName + '.yml'), applicationName + '.yml')
0268 
0269     oldcwd = os.getcwd()
0270     os.chdir(srcPath)
0271     for file in glob.iglob(applicationName + '/**', recursive=True):
0272         archive.write(file, file)
0273     os.chdir(oldcwd)
0274 
0275 # Generate metadata for the given appstream and desktop files
0276 def processAppstreamFile(appstreamFileName, desktopFileName, iconBaseName):
0277     # appstreamFileName has the form <id>.appdata.xml or <id>.metainfo.xml, so we
0278     # have to strip off two extensions
0279     applicationName = os.path.splitext(os.path.splitext(os.path.basename(appstreamFileName))[0])[0]
0280 
0281     data = {}
0282     # Within this file we look at every entry, and where possible try to export it's content so we can use it later
0283     appstreamFile = open(appstreamFileName, "rb")
0284     root = ET.fromstring(appstreamFile.read())
0285 
0286     allLanguages = set()
0287     listAllLanguages(root, allLanguages)
0288 
0289     for child in root:
0290         # Make sure we start with a blank slate for this entry
0291         output = {}
0292 
0293         # Grab the name of this particular attribute we're looking at
0294         # Within the Fastlane specification, it is possible to have several items with the same name but as different types
0295         # We therefore include this within our extracted name for the attribute to differentiate them
0296         tag = child.tag
0297         if 'type' in child.attrib:
0298             tag += '-' + child.attrib['type']
0299 
0300         # Have we found some information already for this particular attribute?
0301         if tag in data:
0302             output = data[tag]
0303 
0304         # Are we dealing with category information here?
0305         # If so, then we need to look into this items children to find out all the categories this APK belongs in
0306         if tag == 'categories':
0307             cats = []
0308             for x in child:
0309                 cats.append(x.text)
0310             output = { None: cats }
0311 
0312         # screenshot links
0313         elif tag == 'screenshots':
0314             output = []
0315             for screenshot in child:
0316                 if screenshot.tag == 'screenshot':
0317                     for image in screenshot:
0318                         if image.tag == 'image':
0319                             output.append(image.text)
0320 
0321         # Otherwise this is just textual information we need to extract
0322         else:
0323             readText(child, output, allLanguages)
0324 
0325         # Save the information we've gathered!
0326         data[tag] = output
0327 
0328     applyLanguageFallback(data['name'], allLanguages)
0329     applyLanguageFallback(data['summary'], allLanguages)
0330     applyLanguageFallback(data['description'], allLanguages)
0331 
0332     # Did we find any categories?
0333     # Sometimes we don't find any within the Fastlane information, but without categories the F-Droid store isn't of much use
0334     # In the event this happens, fallback to the *.desktop file for the application to see if it can provide any insight.
0335     if not 'categories' in data and desktopFileName:
0336         # Parse the XDG format *.desktop file, and extract the categories within it
0337         desktopFile = xdg.DesktopEntry.DesktopEntry(desktopFileName)
0338         data['categories'] = { None: desktopFile.getCategories() }
0339 
0340     # Try to figure out the source repository
0341     if arguments.source and os.path.exists(os.path.join(arguments.source, '.git')):
0342         upstream_ref = subprocess.check_output(['git', 'rev-parse', '--symbolic-full-name', '@{u}'], cwd=arguments.source).decode('utf-8')
0343         remote = upstream_ref.split('/')[2]
0344         output = subprocess.check_output(['git', 'remote', 'get-url', remote], cwd=arguments.source).decode('utf-8')
0345         data['source-repo'] = output.strip()
0346 
0347     # write meta data
0348     createFastlaneFile( applicationName, "title.txt", data['name'] )
0349     createFastlaneFile( applicationName, "short_description.txt", data['summary'] )
0350     createFastlaneFile( applicationName, "full_description.txt", data['description'] )
0351     createYml(applicationName, data)
0352 
0353     # cleanup old image files before collecting new ones
0354     imagePath = os.path.join(arguments.output, 'metadata',  applicationName, 'en-US', 'images')
0355     shutil.rmtree(imagePath, ignore_errors=True)
0356     processLocalImages(applicationName, data)
0357     downloadScreenshots(applicationName, data)
0358     findIcon(applicationName, iconBaseName)
0359 
0360     # put the result in an archive file for easier use by Jenkins
0361     createMetadataArchive(applicationName)
0362 
0363 # scan source directory for manifests/metadata we can work with
0364 def scanSourceDir():
0365     files = glob.iglob(arguments.source + "/**/AndroidManifest.xml*", recursive=True)
0366     for file in files:
0367         # third-party libraries might contain AndroidManifests which we are not interested in
0368         if "3rdparty" in file:
0369             continue
0370 
0371         # find application id from manifest files
0372         root = ET.parse(file)
0373         appname = root.getroot().attrib['package']
0374         is_app = False
0375         prefix = '{http://schemas.android.com/apk/res/android}'
0376         for md in root.findall("application/activity/meta-data"):
0377             if md.attrib[prefix + 'name'] == 'android.app.lib_name':
0378                 is_app = True
0379 
0380         if not appname or not is_app:
0381             continue
0382 
0383         iconBaseName = None
0384         for elem in root.findall('application'):
0385             if prefix + 'icon' in elem.attrib:
0386                 iconBaseName = elem.attrib[prefix + 'icon'].split('/')[-1]
0387 
0388         # now that we have the app id, look for matching appdata/desktop files
0389         appdataFiles = glob.glob(arguments.source + "/**/" + appname + ".metainfo.xml", recursive=True)
0390         appdataFiles.extend(glob.glob(arguments.source + "/**/" + appname + ".appdata.xml", recursive=True))
0391         appdataFile = None
0392         for f in appdataFiles:
0393             appdataFile = f
0394             break
0395         if not appdataFile:
0396             continue
0397 
0398         desktopFiles = glob.iglob(arguments.source + "/**/" + appname + ".desktop", recursive=True)
0399         desktopFile = None
0400         for f in desktopFiles:
0401             desktopFile = f
0402             break
0403 
0404         processAppstreamFile(appdataFile, desktopFile, iconBaseName)
0405 
0406 
0407 ### Script Commences
0408 
0409 # Parse the command line arguments we've been given
0410 parser = argparse.ArgumentParser(description='Generate fastlane metadata for Android apps from appstream metadata')
0411 parser.add_argument('--appstream', type=str, required=False, help='Appstream file to extract metadata from')
0412 parser.add_argument('--desktop', type=str, required=False, help='Desktop file to extract additional metadata from')
0413 parser.add_argument('--source', type=str, required=False, help='Source directory to find metadata in')
0414 parser.add_argument('--output', type=str, required=True, help='Path to which the metadata output should be written to')
0415 arguments = parser.parse_args()
0416 
0417 # ensure the output path exists
0418 os.makedirs(arguments.output, exist_ok=True)
0419 
0420 # if we have an appstream file explicitly specified, let's use that one
0421 if arguments.appstream and os.path.exists(arguments.appstream):
0422     processAppstreamFile(arguments.appstream, arguments.desktop)
0423     sys.exit(0)
0424 
0425 # else, look in the source dir for appstream/desktop files
0426 # this follows roughly what get-apk-args from binary factory does
0427 if arguments.source and os.path.exists(arguments.source):
0428     scanSourceDir()
0429     sys.exit(0)
0430 
0431 # else: missing arguments
0432 print("Either one of --appstream or --source have to be provided!")
0433 sys.exit(1)