File indexing completed on 2024-04-21 03:52:24
0001 #!/usr/bin/env python3 0002 # 0003 # SPDX-FileCopyrightText: 2018-2020 Aleix Pol Gonzalez <aleixpol@kde.org> 0004 # SPDX-FileCopyrightText: 2019-2020 Ben Cooksley <bcooksley@kde.org> 0005 # SPDX-FileCopyrightText: 2020 Volker Krause <vkrause@kde.org> 0006 # 0007 # SPDX-License-Identifier: GPL-2.0-or-later 0008 # 0009 # Generates fastlane metadata for Android apps from appstream files. 0010 # 0011 0012 import argparse 0013 import glob 0014 import io 0015 import os 0016 import re 0017 import requests 0018 import shutil 0019 import subprocess 0020 import sys 0021 import tempfile 0022 import xdg.DesktopEntry 0023 import xml.etree.ElementTree as ET 0024 import yaml 0025 import zipfile 0026 0027 # Constants used in this script 0028 # map KDE's translated language codes to those expected by Android 0029 # see https://f-droid.org/en/docs/Translation_and_Localization/ 0030 # F-Droid is more tolerant than the Play Store here, the latter rejects anything not exactly matching its known codes 0031 # Android does do the expected fallbacks, so the seemingly "too specific" mappings here are still working as expected 0032 # see https://developer.android.com/guide/topics/resources/multilingual-support#resource-resolution-examples 0033 languageMap = { 0034 None: "en-US", 0035 "ast": None, # not supported by Google Play for meta data 0036 "ca-valencia": None, # not supported by Android 0037 "cs": "cs-CZ", 0038 "de": "de-DE", 0039 "eo": None, # neither supported by Android nor by Google Play for meta data 0040 "es": "es-ES", 0041 "eu": "eu-ES", 0042 "fi": "fi-FI", 0043 "fr": "fr-FR", 0044 "gl": "gl-ES", 0045 "ia": None, # not supported by Google Play for meta data 0046 "it": "it-IT", 0047 "ka": "ka-GE", 0048 "ko": "ko-KR", 0049 "nl": "nl-NL", 0050 "pl": "pl-PL", 0051 "pt": "pt-PT", 0052 "ru": "ru-RU", 0053 "sr": "sr-Cyrl-RS", 0054 "sr@latin": "sr-Latn-RS", 0055 "sv": "sv-SE", 0056 "tr": "tr-TR", 0057 'x-test': None 0058 } 0059 0060 # The subset of supported rich text tags in F-Droid and Google Play 0061 # - see https://f-droid.org/en/docs/All_About_Descriptions_Graphics_and_Screenshots/ for F-Droid 0062 # - Google Play doesn't support lists 0063 supportedRichTextTags = { 'b', 'u', 'i' } 0064 0065 # List all translated languages present in an Appstream XML file 0066 def listAllLanguages(root, langs): 0067 for elem in root: 0068 lang = elem.get('{http://www.w3.org/XML/1998/namespace}lang') 0069 if not lang in langs: 0070 langs.add(lang) 0071 listAllLanguages(elem, langs) 0072 0073 # Apply language fallback to a map of translations 0074 def applyLanguageFallback(data, allLanguages): 0075 for l in allLanguages: 0076 if not l in data or not data[l] or len(data[l]) == 0: 0077 data[l] = data[None] 0078 0079 # Android appdata.xml textual item parser 0080 # This function handles reading standard text entries within an Android appdata.xml file 0081 # In particular, it handles splitting out the various translations, and converts some HTML to something which F-Droid can make use of 0082 # We have to handle incomplete translations both on top-level and intermediate tags, 0083 # and fall back to the English default text where necessary. 0084 def readText(elem, found, allLanguages): 0085 # Determine the language this entry is in 0086 lang = elem.get('{http://www.w3.org/XML/1998/namespace}lang') 0087 0088 # Do we have any text for this language yet? 0089 # If not, get everything setup 0090 for l in allLanguages: 0091 if not l in found: 0092 found[l] = "" 0093 0094 # If there is text available, we'll want to extract it 0095 # Additionally, if this element has any children, make sure we read those as well 0096 if elem.tag in supportedRichTextTags: 0097 if (elem.text and elem.text.strip()) or lang: 0098 found[lang] += '<' + elem.tag + '>' 0099 else: 0100 for l in allLanguages: 0101 found[l] += '<' + elem.tag + '>' 0102 elif elem.tag == 'li': 0103 found[lang] += 'ยท ' 0104 0105 if elem.text and elem.text.strip(): 0106 found[lang] += elem.text 0107 0108 subOutput = {} 0109 for child in elem: 0110 if not child.get('{http://www.w3.org/XML/1998/namespace}lang') and len(subOutput) > 0: 0111 applyLanguageFallback(subOutput, allLanguages) 0112 for l in allLanguages: 0113 found[l] += subOutput[l] 0114 subOutput = {} 0115 readText(child, subOutput, allLanguages) 0116 if len(subOutput) > 0: 0117 applyLanguageFallback(subOutput, allLanguages) 0118 for l in allLanguages: 0119 found[l] += subOutput[l] 0120 0121 if elem.tag in supportedRichTextTags: 0122 if (elem.text and elem.text.strip()) or lang: 0123 found[lang] += '</' + elem.tag + '>' 0124 else: 0125 for l in allLanguages: 0126 found[l] += '</' + elem.tag + '>' 0127 0128 # Finally, if this element is a HTML Paragraph (p) or HTML List Item (li) make sure we add a new line for presentation purposes 0129 if elem.tag == 'li' or elem.tag == 'p': 0130 found[lang] += "\n" 0131 0132 0133 # Create the various Fastlane format files per the information we've previously extracted 0134 # These files are laid out following the Fastlane specification (links below) 0135 # https://github.com/fastlane/fastlane/blob/2.28.7/supply/README.md#images-and-screenshots 0136 # https://docs.fastlane.tools/actions/supply/ 0137 def createFastlaneFile( applicationName, filenameToPopulate, fileContent ): 0138 # Go through each language and content pair we've been given 0139 for lang, text in fileContent.items(): 0140 # First, do we need to amend the language id, to turn the Android language ID into something more F-Droid/Fastlane friendly? 0141 languageCode = languageMap.get(lang, lang) 0142 if not languageCode: 0143 continue 0144 0145 # Next we need to determine the path to the directory we're going to be writing the data into 0146 repositoryBasePath = arguments.output 0147 path = os.path.join( repositoryBasePath, 'metadata', applicationName, languageCode ) 0148 0149 # Make sure the directory exists 0150 os.makedirs(path, exist_ok=True) 0151 0152 # Now write out file contents! 0153 with open(path + '/' + filenameToPopulate, 'w') as f: 0154 f.write(text.strip()) # trim whitespaces, to avoid spurious differences after a Google Play roundtrip 0155 0156 # Create the summary appname.yml file used by F-Droid to summarise this particular entry in the repository 0157 # see https://f-droid.org/en/docs/Build_Metadata_Reference/ 0158 def createYml(appname, data): 0159 # Prepare to retrieve the existing information 0160 info = {} 0161 0162 # Determine the path to the appname.yml file 0163 repositoryBasePath = arguments.output 0164 path = os.path.join( repositoryBasePath, 'metadata', appname + '.yml' ) 0165 0166 # Update the categories first 0167 # Now is also a good time to add 'KDE' to the list of categories as well 0168 if 'categories' in data: 0169 info['Categories'] = data['categories'][None] + ['KDE'] 0170 else: 0171 info['Categories'] = ['KDE'] 0172 0173 # Update the general summary as well 0174 info['Summary'] = data['summary'][None] 0175 0176 # Check to see if we have a Homepage... 0177 if 'url-homepage' in data: 0178 info['WebSite'] = data['url-homepage'][None] 0179 0180 # What about a bug tracker? 0181 if 'url-bugtracker' in data: 0182 info['IssueTracker'] = data['url-bugtracker'][None] 0183 0184 if 'project_license' in data: 0185 info["License"] = data['project_license'][None] 0186 0187 if 'source-repo' in data: 0188 info['SourceCode'] = data['source-repo'] 0189 0190 if 'url-donation' in data: 0191 info['Donate'] = data['url-donation'][None] 0192 else: 0193 info['Donate'] = 'https://kde.org/community/donations/' 0194 0195 # static data 0196 info['Translation'] = 'https://l10n.kde.org/' 0197 0198 # Finally, with our updates completed, we can save the updated appname.yml file back to disk 0199 with open(path, 'w') as output: 0200 yaml.dump(info, output, default_flow_style=False) 0201 0202 # Integrates locally existing image assets into the metadata 0203 def processLocalImages(applicationName, data): 0204 if not os.path.exists(os.path.join(arguments.source, 'fastlane')): 0205 return 0206 0207 outPath = os.path.abspath(arguments.output); 0208 oldcwd = os.getcwd() 0209 os.chdir(os.path.join(arguments.source, 'fastlane')) 0210 0211 imageFiles = glob.glob('metadata/**/*.png', recursive=True) 0212 imageFiles.extend(glob.glob('metadata/**/*.jpg', recursive=True)) 0213 for image in imageFiles: 0214 # noramlize single- vs multi-app layouts 0215 imageDestName = image.replace('metadata/android', 'metadata/' + applicationName) 0216 0217 # copy image 0218 os.makedirs(os.path.dirname(os.path.join(outPath, imageDestName)), exist_ok=True) 0219 shutil.copy(image, os.path.join(outPath, imageDestName)) 0220 0221 # if the source already contains screenshots, those override whatever we found in the appstream file 0222 if 'phoneScreenshots' in image: 0223 data['screenshots'] = {} 0224 0225 os.chdir(oldcwd) 0226 0227 # Attempt to find the application icon if we haven't gotten that explicitly from processLocalImages 0228 def findIcon(applicationName, iconBaseName): 0229 iconPath = os.path.join(arguments.output, 'metadata', applicationName, 'en-US', 'images', 'icon.png') 0230 if os.path.exists(iconPath): 0231 return 0232 0233 oldcwd = os.getcwd() 0234 os.chdir(arguments.source) 0235 0236 iconFiles = glob.glob(f"**/{iconBaseName}-playstore.png", recursive=True) 0237 for icon in iconFiles: 0238 os.makedirs(os.path.dirname(iconPath), exist_ok=True) 0239 shutil.copy(icon, iconPath) 0240 break 0241 0242 os.chdir(oldcwd) 0243 0244 # Download screenshots referenced in the appstream data 0245 # see https://f-droid.org/en/docs/All_About_Descriptions_Graphics_and_Screenshots/ 0246 def downloadScreenshots(applicationName, data): 0247 if not 'screenshots' in data: 0248 return 0249 0250 path = os.path.join(arguments.output, 'metadata', applicationName, 'en-US', 'images', 'phoneScreenshots') 0251 os.makedirs(path, exist_ok=True) 0252 0253 i = 1 # number screenshots starting at 1 rather than 0 to match what the fastlane tool does 0254 for screenshot in data['screenshots']: 0255 fileName = str(i) + '-' + screenshot[screenshot.rindex('/') + 1:] 0256 r = requests.get(screenshot) 0257 if r.status_code < 400: 0258 with open(os.path.join(path, fileName), 'wb') as f: 0259 f.write(r.content) 0260 i += 1 0261 0262 # Put all metadata for the given application name into an archive 0263 # We need this to easily transfer the entire metadata to the signing machine for integration 0264 # into the F-Droid nightly repository 0265 def createMetadataArchive(applicationName): 0266 srcPath = os.path.join(arguments.output, 'metadata') 0267 zipFileName = os.path.join(srcPath, 'fastlane-' + applicationName + '.zip') 0268 if os.path.exists(zipFileName): 0269 os.unlink(zipFileName) 0270 archive = zipfile.ZipFile(zipFileName, 'w') 0271 archive.write(os.path.join(srcPath, applicationName + '.yml'), applicationName + '.yml') 0272 0273 oldcwd = os.getcwd() 0274 os.chdir(srcPath) 0275 for file in glob.iglob(applicationName + '/**', recursive=True): 0276 archive.write(file, file) 0277 os.chdir(oldcwd) 0278 0279 # Generate metadata for the given appstream and desktop files 0280 def processAppstreamFile(appstreamFileName, desktopFileName, iconBaseName): 0281 # appstreamFileName has the form <id>.appdata.xml or <id>.metainfo.xml, so we 0282 # have to strip off two extensions 0283 applicationName = os.path.splitext(os.path.splitext(os.path.basename(appstreamFileName))[0])[0] 0284 0285 data = {} 0286 # Within this file we look at every entry, and where possible try to export it's content so we can use it later 0287 appstreamFile = open(appstreamFileName, "rb") 0288 root = ET.fromstring(appstreamFile.read()) 0289 0290 allLanguages = set() 0291 listAllLanguages(root, allLanguages) 0292 0293 for child in root: 0294 # Make sure we start with a blank slate for this entry 0295 output = {} 0296 0297 # Grab the name of this particular attribute we're looking at 0298 # Within the Fastlane specification, it is possible to have several items with the same name but as different types 0299 # We therefore include this within our extracted name for the attribute to differentiate them 0300 tag = child.tag 0301 if 'type' in child.attrib: 0302 tag += '-' + child.attrib['type'] 0303 0304 # Have we found some information already for this particular attribute? 0305 if tag in data: 0306 output = data[tag] 0307 0308 # Are we dealing with category information here? 0309 # If so, then we need to look into this items children to find out all the categories this APK belongs in 0310 if tag == 'categories': 0311 cats = [] 0312 for x in child: 0313 cats.append(x.text) 0314 output = { None: cats } 0315 0316 # screenshot links 0317 elif tag == 'screenshots': 0318 output = [] 0319 for screenshot in child: 0320 if screenshot.tag == 'screenshot': 0321 for image in screenshot: 0322 if image.tag == 'image': 0323 output.append(image.text) 0324 0325 # Otherwise this is just textual information we need to extract 0326 else: 0327 readText(child, output, allLanguages) 0328 0329 # Save the information we've gathered! 0330 data[tag] = output 0331 0332 applyLanguageFallback(data['name'], allLanguages) 0333 applyLanguageFallback(data['summary'], allLanguages) 0334 applyLanguageFallback(data['description'], allLanguages) 0335 0336 # Did we find any categories? 0337 # Sometimes we don't find any within the Fastlane information, but without categories the F-Droid store isn't of much use 0338 # In the event this happens, fallback to the *.desktop file for the application to see if it can provide any insight. 0339 if not 'categories' in data and desktopFileName: 0340 # Parse the XDG format *.desktop file, and extract the categories within it 0341 desktopFile = xdg.DesktopEntry.DesktopEntry(desktopFileName) 0342 data['categories'] = { None: desktopFile.getCategories() } 0343 0344 # Try to figure out the source repository 0345 if arguments.source and os.path.exists(os.path.join(arguments.source, '.git')): 0346 upstream_ref = subprocess.check_output(['git', 'rev-parse', '--symbolic-full-name', '@{u}'], cwd=arguments.source).decode('utf-8') 0347 remote = upstream_ref.split('/')[2] 0348 output = subprocess.check_output(['git', 'remote', 'get-url', remote], cwd=arguments.source).decode('utf-8') 0349 data['source-repo'] = output.strip() 0350 0351 # write meta data 0352 createFastlaneFile( applicationName, "title.txt", data['name'] ) 0353 createFastlaneFile( applicationName, "short_description.txt", data['summary'] ) 0354 createFastlaneFile( applicationName, "full_description.txt", data['description'] ) 0355 createYml(applicationName, data) 0356 0357 # cleanup old image files before collecting new ones 0358 imagePath = os.path.join(arguments.output, 'metadata', applicationName, 'en-US', 'images') 0359 shutil.rmtree(imagePath, ignore_errors=True) 0360 processLocalImages(applicationName, data) 0361 downloadScreenshots(applicationName, data) 0362 findIcon(applicationName, iconBaseName) 0363 0364 # put the result in an archive file for easier use by Jenkins 0365 createMetadataArchive(applicationName) 0366 0367 # scan source directory for manifests/metadata we can work with 0368 def scanSourceDir(): 0369 files = glob.iglob(arguments.source + "/**/AndroidManifest.xml*", recursive=True) 0370 for file in files: 0371 # third-party libraries might contain AndroidManifests which we are not interested in 0372 if "3rdparty" in file: 0373 continue 0374 0375 # find application id from manifest files 0376 root = ET.parse(file) 0377 appname = root.getroot().attrib['package'] 0378 is_app = False 0379 prefix = '{http://schemas.android.com/apk/res/android}' 0380 for md in root.findall("application/activity/meta-data"): 0381 if md.attrib[prefix + 'name'] == 'android.app.lib_name': 0382 is_app = True 0383 0384 if not appname or not is_app: 0385 continue 0386 0387 iconBaseName = None 0388 for elem in root.findall('application'): 0389 if prefix + 'icon' in elem.attrib: 0390 iconBaseName = elem.attrib[prefix + 'icon'].split('/')[-1] 0391 0392 # now that we have the app id, look for matching appdata/desktop files 0393 appdataFiles = glob.glob(arguments.source + "/**/" + appname + ".metainfo.xml", recursive=True) 0394 appdataFiles.extend(glob.glob(arguments.source + "/**/" + appname + ".appdata.xml", recursive=True)) 0395 appdataFile = None 0396 for f in appdataFiles: 0397 appdataFile = f 0398 break 0399 if not appdataFile: 0400 continue 0401 0402 desktopFiles = glob.iglob(arguments.source + "/**/" + appname + ".desktop", recursive=True) 0403 desktopFile = None 0404 for f in desktopFiles: 0405 desktopFile = f 0406 break 0407 0408 processAppstreamFile(appdataFile, desktopFile, iconBaseName) 0409 0410 0411 ### Script Commences 0412 0413 # Parse the command line arguments we've been given 0414 parser = argparse.ArgumentParser(description='Generate fastlane metadata for Android apps from appstream metadata') 0415 parser.add_argument('--appstream', type=str, required=False, help='Appstream file to extract metadata from') 0416 parser.add_argument('--desktop', type=str, required=False, help='Desktop file to extract additional metadata from') 0417 parser.add_argument('--source', type=str, required=False, help='Source directory to find metadata in') 0418 parser.add_argument('--output', type=str, required=True, help='Path to which the metadata output should be written to') 0419 arguments = parser.parse_args() 0420 0421 # ensure the output path exists 0422 os.makedirs(arguments.output, exist_ok=True) 0423 0424 # if we have an appstream file explicitly specified, let's use that one 0425 if arguments.appstream and os.path.exists(arguments.appstream): 0426 processAppstreamFile(arguments.appstream, arguments.desktop) 0427 sys.exit(0) 0428 0429 # else, look in the source dir for appstream/desktop files 0430 # this follows roughly what get-apk-args from binary factory does 0431 if arguments.source and os.path.exists(arguments.source): 0432 scanSourceDir() 0433 sys.exit(0) 0434 0435 # else: missing arguments 0436 print("Either one of --appstream or --source have to be provided!") 0437 sys.exit(1)