File indexing completed on 2024-11-10 08:12:43
0001 #!/usr/bin/env python3 0002 # -*- coding: UTF-8 -*- 0003 0004 import copy 0005 from difflib import SequenceMatcher 0006 import filecmp 0007 import hashlib 0008 import locale 0009 import os 0010 import re 0011 import shutil 0012 import sys 0013 import time 0014 from functools import reduce 0015 0016 try: 0017 import fallback_import_paths 0018 except: 0019 pass 0020 0021 from pology import version, _, n_, t_, PologyError 0022 from pology.ascript import collect_ascription_associations 0023 from pology.ascript import collect_ascription_history 0024 from pology.ascript import make_ascription_selector 0025 from pology.catalog import Catalog 0026 from pology.header import Header, format_datetime 0027 from pology.message import Message, MessageUnsafe 0028 from pology.colors import ColorOptionParser 0029 from pology.fsops import str_to_unicode, unicode_to_str 0030 from pology.fsops import mkdirpath, assert_system, collect_system 0031 from pology.fsops import getucwd, join_ncwd 0032 from pology.fsops import collect_paths_cmdline, build_path_selector 0033 from pology.fsops import exit_on_exception 0034 from pology.merge import merge_pofile 0035 from pology.monitored import Monpair, Monlist 0036 from pology.msgreport import report_on_msg 0037 from pology.report import report, error, warning, format_item_list 0038 from pology.report import init_file_progress 0039 from pology.stdcmdopt import add_cmdopt_incexc, add_cmdopt_filesfrom 0040 from pology.vcs import make_vcs 0041 from pology.wrap import select_field_wrapping 0042 0043 0044 SUMMIT_ID = "+" # must not start with word-character (\w) 0045 0046 0047 def main (): 0048 0049 locale.setlocale(locale.LC_ALL, "") 0050 0051 # Setup options and parse the command line. 0052 usage = _("@info command usage", 0053 "\n" 0054 " %(cmd)s [OPTIONS] CFGFILE LANG OPMODE [PARTIAL...]\n" 0055 " (if there is no '%(cfgfile)s' file in a parent directory)\n" 0056 " %(cmd)s [OPTIONS] OPMODE [PARTIAL...]\n" 0057 " (if there is a '%(cfgfile)s' file in a parent directory)", 0058 cmd="%prog", cfgfile="summit-config") 0059 desc = _("@info command description", 0060 "Translate PO files spread across different branches " 0061 "in a unified fashion.") 0062 ver = _("@info command version", 0063 "%(cmd)s (Pology) %(version)s\n" 0064 "Copyright © 2007, 2008, 2009, 2010 " 0065 "Chusslove Illich (Часлав Илић) <%(email)s>", 0066 cmd="%prog", version=version(), email="caslav.ilic@gmx.net") 0067 0068 opars = ColorOptionParser(usage=usage, description=desc, version=ver) 0069 opars.add_option( 0070 "-a", "--asc-filter", 0071 action="store", dest="asc_filter", default=None, 0072 help=_("@info command line option description", 0073 "Apply a non-default ascription filter on scatter.")) 0074 opars.add_option( 0075 "--create", 0076 action="store_true", dest="create", default=False, 0077 help=_("@info command line option description", 0078 "Allow creation of new summit catalogs.")) 0079 opars.add_option( 0080 "--force", 0081 action="store_true", dest="force", default=False, 0082 help=_("@info command line option description", 0083 "Force some operations that are normally not advised.")) 0084 opars.add_option( 0085 "-q", "--quiet", 0086 action="store_true", dest="quiet", default=False, 0087 help=_("@info command line option description", 0088 "Output less detailed progress info.")) 0089 opars.add_option( 0090 "-v", "--verbose", 0091 action="store_true", dest="verbose", default=False, 0092 help=_("@info command line option description", 0093 "Output more detailed progress info")) 0094 add_cmdopt_filesfrom(opars) 0095 add_cmdopt_incexc(opars) 0096 0097 options, free_args = opars.parse_args(str_to_unicode(sys.argv[1:])) 0098 0099 # Look for the config file through parent directories. 0100 parent = getucwd() 0101 cfgpath = None 0102 while True: 0103 for cfgname in ("summit-config",): 0104 cfgpath1 = os.path.join(parent, cfgname) 0105 if os.path.isfile(cfgpath1): 0106 cfgpath = cfgpath1 0107 break 0108 if cfgpath: 0109 break 0110 pparent = parent 0111 parent = os.path.dirname(parent) 0112 if parent == pparent: 0113 break 0114 0115 # If config file not found, expect it and language as arguments. 0116 if not cfgpath: 0117 if len(free_args) < 1: 0118 error(_("@info", 0119 "Summit configuration file neither found " 0120 "as '%(cfgfile)s' in parent directories, " 0121 "nor given in command line.", 0122 cfgfile="summit-config")) 0123 cfgpath = free_args.pop(0) 0124 if not os.path.isfile(cfgpath): 0125 error(_("@info", 0126 "Summit configuration file '%(file)s' does not exist.", 0127 file=cfgpath)) 0128 0129 if len(free_args) < 1: 0130 error(_("@info", 0131 "Language code not given.")) 0132 lang = free_args.pop(0) 0133 else: 0134 lang = None 0135 # ...will be read from config file. 0136 0137 if len(free_args) < 1: 0138 error(_("@info", "Operation mode not given.")) 0139 opmodes = free_args.pop(0).split(",") 0140 opmodes_uniq = [] 0141 for opmode in opmodes: 0142 if opmode not in opmodes_uniq: 0143 if opmode not in ("gather", "scatter", "merge", "deps"): 0144 error(_("@info", 0145 "Unknown operation mode '%(mode)s'.", 0146 mode=opmode)) 0147 opmodes_uniq.append(opmode) 0148 opmodes = opmodes_uniq 0149 0150 # Could use some speedup. 0151 try: 0152 import psyco 0153 psyco.full() 0154 except ImportError: 0155 pass 0156 0157 # Read project definition. 0158 project = Project(lang, opmodes, options) 0159 project.include(cfgpath) 0160 0161 # In summit-over-templates mode, determine if templates are dynamic. 0162 project.templates_dynamic = ( project.over_templates 0163 and not project.summit.get("topdir_templates")) 0164 0165 # If config file was found in parent directories, 0166 # it should have defined the language itself. 0167 # Otherwise, its language is set to language given in command line. 0168 if not lang: 0169 if not project.lang: 0170 error(_("@info", 0171 "Language code not set in configuration file.")) 0172 lang = project.lang 0173 else: 0174 project.lang = lang 0175 0176 # In summit-over-templates mode, derive special project data 0177 # for implicitly gathering templates on merge. 0178 if project.templates_dynamic and "merge" in project.opmodes: 0179 project.toptions = copy.copy(options) 0180 project.toptions.quiet = True 0181 project.tproject = Project(project.templates_lang, ["gather"], 0182 project.toptions) 0183 project.tproject.include(cfgpath) 0184 project.tproject.templates_dynamic = False 0185 project.tproject.summit_version_control = "none" 0186 project.tproject.summit_wrap = False # performance 0187 project.tproject.summit_fine_wrap = False # performance 0188 tpd = project.tproject.summit.get("topdir_templates") 0189 if tpd is None: 0190 # FIXME: Portability. 0191 tpd = "/tmp/summit-templates-%d" % os.getpid() 0192 project.tproject.summit["topdir"] = tpd 0193 for tb in project.tproject.branches: 0194 tbpd = tb.get("topdir_templates") 0195 if tbpd is not None: 0196 tb["topdir"] = tbpd 0197 project.tproject.lang = project.templates_lang 0198 project.tproject = derive_project_data(project.tproject, 0199 project.toptions, 0200 project.summit["topdir"]) 0201 project.summit["topdir_templates"] = tpd 0202 0203 # Explicit gathering in summit-over-templates mode 0204 # may be useful to check if gathering works. 0205 # Make some adjustments for this to go smoothly. 0206 if ( project.templates_dynamic and "gather" in project.opmodes 0207 and project.lang == project.templates_lang 0208 ): 0209 options.create = True 0210 project.summit["topdir"] = project.summit["topdir_templates"] 0211 project.summit_version_control = "none" 0212 0213 # Derive project data. 0214 project = derive_project_data(project, options) 0215 0216 # Collect partial processing specs and inclusion-exclusion test. 0217 specargs, ffself = collect_paths_cmdline(rawpaths=free_args, 0218 filesfrom=options.files_from, 0219 getsel=True, abort=True) 0220 options.partspecs, options.partbids = collect_partspecs(project, specargs) 0221 if not options.files_from: 0222 # If there was no from-file input and no partial processing specs 0223 # were collected, indicate operation on the whole summit. 0224 if not options.partspecs: 0225 options.partspecs = None 0226 if not options.partbids: 0227 options.partbids = None 0228 cmdself = build_path_selector(incnames=options.include_names, 0229 incpaths=options.include_paths, 0230 excnames=options.exclude_names, 0231 excpaths=options.exclude_paths) 0232 options.selcatf = lambda x: cmdself(x) and ffself(x) 0233 0234 # Invoke the appropriate operations on collected bundles. 0235 for opmode in opmodes: 0236 if options.verbose: 0237 report(_("@info:progress", 0238 "-----> Processing mode: %(mode)s", 0239 mode=opmode)) 0240 if opmode == "gather": 0241 summit_gather(project, options) 0242 elif opmode == "scatter": 0243 summit_scatter(project, options) 0244 elif opmode == "merge": 0245 summit_merge(project, options) 0246 elif opmode == "deps": 0247 summit_deps(project, options) 0248 0249 0250 class Project (object): 0251 0252 def __init__ (self, lang, opmodes, options): 0253 0254 self.__dict__.update({ 0255 "lang" : lang, 0256 "opmodes" : opmodes, 0257 "options" : options, 0258 0259 "summit" : {}, 0260 "branches" : [], 0261 "mappings" : [], 0262 "subdir_mappings" : [], 0263 "subdir_precedence" : [], 0264 0265 "over_templates" : False, 0266 "templates_lang" : "templates", 0267 0268 "summit_wrap" : False, 0269 "summit_fine_wrap" : True, 0270 "summit_fuzzy_merging" : True, 0271 "branches_wrap" : True, 0272 "branches_fine_wrap" : True, 0273 "branches_fuzzy_merging" : True, 0274 0275 "version_control" : "", 0276 "summit_version_control" : "", 0277 "branches_version_control" : "", 0278 0279 "hook_on_scatter_msgstr" : [], 0280 "hook_on_scatter_msg" : [], 0281 "hook_on_scatter_cat" : [], 0282 "hook_on_scatter_file" : [], 0283 "hook_on_scatter_branch": [], 0284 "hook_on_gather_msg" : [], 0285 "hook_on_gather_msg_branch" : [], 0286 "hook_on_gather_cat" : [], 0287 "hook_on_gather_cat_branch" : [], 0288 "hook_on_gather_file" : [], 0289 "hook_on_gather_file_branch" : [], 0290 "hook_on_merge_msg" : [], 0291 "hook_on_merge_head" : [], 0292 "hook_on_merge_cat" : [], 0293 "hook_on_merge_file" : [], 0294 0295 "header_propagate_fields" : [], 0296 "header_skip_fields_on_scatter" : [], 0297 0298 "vivify_on_merge" : False, 0299 "vivify_w_translator" : "Simulacrum", 0300 "vivify_w_langteam" : "Nevernessian", 0301 "vivify_w_language" : "", 0302 "vivify_w_charset" : "UTF-8", 0303 "vivify_w_plurals" : "", 0304 0305 "compendium_on_merge" : "", 0306 "compendium_fuzzy_exact" : False, 0307 "compendium_min_words_exact" : 0, 0308 0309 "merge_min_adjsim_fuzzy" : 0.0, 0310 "merge_rebase_fuzzy" : False, 0311 0312 "scatter_min_completeness" : 0.0, 0313 "scatter_acc_completeness" : 0.0, 0314 0315 "ascription_filters" : [], 0316 "ascription_history_filter" : None, 0317 }) 0318 self.__dict__["locked"] = False 0319 0320 self.inclusion_trail = [] 0321 0322 0323 def __setattr__ (self, att, val): 0324 0325 # TODO: Do extensive checks. 0326 if self.locked and att not in self.__dict__: 0327 error(_("@info", 0328 "Unknown summit configuration field '%(field)s'.", 0329 field=att)) 0330 self.__dict__[att] = val 0331 0332 0333 def relpath (self, path): 0334 0335 rootdir = os.path.dirname(self.inclusion_trail[-1]) 0336 if not os.path.isabs(path): 0337 path = join_ncwd(rootdir, path) 0338 0339 return path 0340 0341 0342 # FIXME: Temporary for backward compatibility, remove at some point. 0343 def resolve_path_rooted (self, path): 0344 0345 return self.relpath(path) 0346 0347 0348 def include (self, path): 0349 0350 path = os.path.abspath(path) 0351 if path in self.inclusion_trail: 0352 error(_("@info", 0353 "Circular inclusion of '%(file)s' attempted " 0354 "in summit configuration.", 0355 file=path)) 0356 self.inclusion_trail.append(path) 0357 self.locked = True 0358 with open(path) as input: 0359 code = input.read() 0360 exec(code, {"S" : self}) 0361 self.locked = False 0362 self.inclusion_trail.pop() 0363 0364 0365 def derive_project_data (project, options, nwgrefpath=None): 0366 0367 p = project # shortcut 0368 0369 # Create summit object from summit dictionary. 0370 class Summit: pass 0371 s = Summit() 0372 sd = p.summit 0373 s.id = SUMMIT_ID 0374 s.by_lang = False 0375 s.topdir = sd.pop("topdir", None) 0376 s.topdir_templates = sd.pop("topdir_templates", None) 0377 # Assert that there are no misnamed keys in the dictionary. 0378 if sd: 0379 error(_("@info", 0380 "Unknown keys in summit configuration: %(keylist)s.", 0381 keylist=format_item_list(list(sd.keys())))) 0382 # Assert that all necessary fields in summit specification exist. 0383 if s.topdir is None: 0384 error(_("@info", 0385 "Top directory not set in summit configuration.")) 0386 s.split_path = None # needed only on some checks later 0387 p.summit = s 0388 0389 # Create branch objects from branch dictionaries. 0390 class Branch: pass 0391 branches = [] 0392 for bd in p.branches: 0393 b = Branch() 0394 branches.append(b) 0395 0396 b.id = bd.pop("id", None) 0397 b.topdir = bd.pop("topdir", None) 0398 b.topdir_templates = bd.pop("topdir_templates", None) 0399 # If operation is performed on templates and branch template directory 0400 # is defined, override plain branch directory with it. 0401 if p.lang == p.templates_lang and b.topdir_templates is not None: 0402 b.topdir = b.topdir_templates 0403 b.by_lang = bd.pop("by_lang", False) 0404 if b.by_lang and isinstance(b.by_lang, bool): 0405 b.by_lang = project.lang 0406 # If separate templates directory is not defined in by-language mode, 0407 # set it to same as catalogs directory. 0408 if b.by_lang and b.topdir_templates is None: 0409 b.topdir_templates = b.topdir 0410 b.scatter_create_filter = bd.pop("scatter_create_filter", None) 0411 b.skip_version_control = bd.pop("skip_version_control", False) 0412 # FIXME: merge_locally retained for backward compatibility, 0413 # replace at some point with b.merge = bd.pop("merge", False). 0414 b.merge = bd.pop("merge", None) 0415 if b.merge is None: 0416 b.merge = bd.pop("merge_locally", False) 0417 b.split_path, b.join_path = bd.pop("transform_path", (None, None)) 0418 b.insert_nosim = bd.pop("insert_nosim", False) 0419 0420 # Assemble include-exclude functions. 0421 includes = bd.pop("includes", []) 0422 excludes = bd.pop("excludes", []) 0423 0424 def regex_to_func (rxstr): 0425 try: 0426 rx = re.compile(rxstr, re.U) 0427 except: 0428 error(_("@info", 0429 "Invalid regular expression '%(regex)s' " 0430 "in include-exclude specification " 0431 "of branch '%(branch)s'.", 0432 branch=b.id, regex=rxstr)) 0433 return lambda x: bool(rx.search(x)) 0434 0435 def chain_tests (tests): 0436 testfs = [] 0437 for test in tests: 0438 if isinstance(test, str): 0439 testfs.append(regex_to_func(test)) 0440 elif callable(test): 0441 testfs.append(test) 0442 else: 0443 error(_("@info", 0444 "Invalid test type '%(type)s' " 0445 "in include-exclude specification " 0446 "of branch '%(branch)s'.", 0447 branch=b.id, type=type(test))) 0448 return lambda x: reduce(lambda s, y: s or y(x), testfs, False) 0449 0450 if includes: 0451 includef = chain_tests(includes) 0452 if excludes: 0453 excludef = chain_tests(excludes) 0454 if includes and excludes: 0455 b.ignored = lambda x: not includef(x) or excludef(x) 0456 elif includes: 0457 b.ignored = lambda x: not includef(x) 0458 elif excludes: 0459 b.ignored = lambda x: excludef(x) 0460 else: 0461 b.ignored = lambda x: False 0462 0463 # Assert that there are no misnamed keys in the dictionary. 0464 if bd: 0465 error(_("@info", 0466 "Unknown keys in specification of branch '%(branch)s': " 0467 "%(keylist)s.", 0468 branch=b.id, keylist=format_item_list(list(bd.keys())))) 0469 p.branches = branches 0470 0471 # Assert that all necessary fields in branch specifications exist. 0472 p.branch_ids = [] 0473 for branch in p.branches: 0474 if branch.id is None: 0475 error(_("@info", 0476 "Branch with undefined ID.")) 0477 if branch.id in p.branch_ids: 0478 error(_("@info", 0479 "Non-unique branch ID '%(branch)s'.", 0480 branch=branch.id)) 0481 p.branch_ids.append(branch.id) 0482 if branch.topdir is None: 0483 error(_("@info", 0484 "Top directory not set for branch '%(branch)s'.", 0485 branch=branch.id)) 0486 0487 # Dictionary of branches by branch id. 0488 p.bdict = dict([(x.id, x) for x in p.branches]) 0489 0490 # Create version control operators if given. 0491 p.summit_vcs = None 0492 p.branches_vcs = None 0493 if p.summit_version_control: 0494 p.summit_vcs = make_vcs(p.summit_version_control.lower()) 0495 if p.branches_version_control: 0496 p.branches_vcs = make_vcs(p.branches_version_control.lower()) 0497 if p.version_control: 0498 if p.summit_vcs is None: 0499 p.summit_vcs = make_vcs(p.version_control.lower()) 0500 if p.branches_vcs is None: 0501 p.branches_vcs = make_vcs(p.version_control.lower()) 0502 0503 # Decide wrapping policies. 0504 class D: pass 0505 dummyopt = D() 0506 dummyopt.do_wrap = p.summit_wrap 0507 dummyopt.do_fine_wrap = p.summit_fine_wrap 0508 p.summit_wrapping = select_field_wrapping(cmlopt=dummyopt) 0509 dummyopt.do_wrap = p.branches_wrap 0510 dummyopt.do_fine_wrap = p.branches_fine_wrap 0511 p.branches_wrapping = select_field_wrapping(cmlopt=dummyopt) 0512 0513 # Decide the extension of catalogs. 0514 if p.over_templates and p.lang == p.templates_lang: 0515 catext = ".pot" 0516 else: 0517 catext = ".po" 0518 0519 # Collect catalogs from branches. 0520 p.catalogs = {} 0521 for b in p.branches: 0522 p.catalogs[b.id] = collect_catalogs(b.topdir, catext, 0523 b.by_lang, b.ignored, b.split_path, 0524 project, options) 0525 # ...and from the summit. 0526 p.catalogs[SUMMIT_ID] = collect_catalogs(p.summit.topdir, catext, 0527 None, None, None, 0528 project, options) 0529 if ( p.lang == p.templates_lang and "gather" in p.opmodes 0530 and nwgrefpath is not None 0531 ): 0532 # Also add summit templates which do not actually exist, 0533 # but are going to be created on gather without warnings, 0534 # by reflecting the catalogs found in the given path. 0535 refcats = collect_catalogs(nwgrefpath, ".po", 0536 None, None, None, project, options) 0537 for name, spec in refcats.items(): 0538 if name not in p.catalogs[SUMMIT_ID]: 0539 path, subdir = spec[0] # all summit catalogs unique 0540 tpath = join_ncwd(p.summit.topdir, subdir, name + ".pot") 0541 p.catalogs[SUMMIT_ID][name] = [(tpath, subdir)] 0542 0543 # Resolve ascription filter. 0544 project.ascription_filter = None 0545 for afname, afspec in project.ascription_filters: 0546 if options.asc_filter is None or afname == options.asc_filter: 0547 if isinstance(afspec, str): 0548 afcall = make_ascription_selector([afspec]) 0549 elif isinstance(afspec, (tuple, list)): 0550 afcall = make_ascription_selector(afspec) 0551 elif callable(afspec): 0552 afcall = afspec 0553 else: 0554 error(_("@info", 0555 "Unknown type of definition for " 0556 "ascription filter '%(filt)s'.", 0557 filt=afname)) 0558 project.ascription_filter = afcall 0559 break 0560 if options.asc_filter is not None and project.ascription_filter is None: 0561 error(_("@info", 0562 "Summit configuration does not define " 0563 "ascription filter '%(filt)s'.", 0564 filt=options.asc_filter)) 0565 0566 # Link summit and ascription catalogs. 0567 if project.ascription_filter: 0568 tmp0 = [(x, y[0][0]) for x, y in list(p.catalogs[SUMMIT_ID].items())] 0569 tmp1 = [x[0] for x in tmp0] 0570 tmp2 = collect_ascription_associations([x[1] for x in tmp0]) 0571 tmp3 = list(zip([tmp2[0][0]] * len(tmp1), [x[1] for x in tmp2[0][1]])) 0572 p.aconfs_acatpaths = dict(list(zip(tmp1, tmp3))) 0573 0574 # Assure that summit catalogs are unique. 0575 for name, spec in list(p.catalogs[SUMMIT_ID].items()): 0576 if len(spec) > 1: 0577 fstr = "\n".join([x[0] for x in spec]) 0578 error(_("@info", 0579 "Non-unique summit catalog '%(name)s', found as:\n" 0580 "%(filelist)s", 0581 name=name, filelist=fstr)) 0582 0583 # At scatter in summit-over-static-templates mode, add to the collection 0584 # of branch catalogs any that should be newly created. 0585 p.add_on_scatter = {} 0586 if ( p.over_templates and p.lang != p.templates_lang 0587 and "scatter" in p.opmodes): 0588 0589 # Go through all mappings and collect branch names mapped to 0590 # summit catalogs per branch id and summit name, and vice versa. 0591 mapped_summit_names = {} 0592 mapped_branch_names = {} 0593 for mapping in p.mappings: 0594 branch_id = mapping[0] 0595 branch_name = mapping[1] 0596 summit_names = mapping[2:] 0597 if not branch_id in mapped_summit_names: 0598 mapped_summit_names[branch_id] = {} 0599 if not branch_id in mapped_branch_names: 0600 mapped_branch_names[branch_id] = {} 0601 for summit_name in summit_names: 0602 if not summit_name in mapped_summit_names[branch_id]: 0603 mapped_summit_names[branch_id][summit_name] = [] 0604 mapped_summit_names[branch_id][summit_name].append(branch_name) 0605 if not branch_name in mapped_branch_names[branch_id]: 0606 mapped_branch_names[branch_id][branch_name] = [] 0607 mapped_branch_names[branch_id][branch_name].append(summit_name) 0608 0609 # Go through all branches. 0610 bt_cache = {} 0611 for branch in p.branches: 0612 # Skip this branch if no templates. 0613 if not branch.topdir_templates: 0614 continue 0615 0616 # Collect all templates for this branch. 0617 branch_templates = bt_cache.get(branch.topdir_templates) 0618 if branch_templates is None: 0619 branch_templates = collect_catalogs(branch.topdir_templates, 0620 ".pot", branch.by_lang, 0621 branch.ignored, 0622 branch.split_path, 0623 project, options) 0624 bt_cache[branch.topdir_templates] = branch_templates 0625 0626 # Go through all summit catalogs. 0627 for summit_name in p.catalogs[SUMMIT_ID]: 0628 0629 # Collect names of any catalogs in this branch mapped to 0630 # the current summit catalog. 0631 branch_names = [] 0632 if ( branch.id in mapped_summit_names 0633 and summit_name in mapped_summit_names[branch.id]): 0634 branch_names = mapped_summit_names[branch.id][summit_name] 0635 # Unconditionally add summit name as one possible branch name, 0636 # since otherwise a mapped branch catalog could shadow 0637 # a direct branch catalog. 0638 branch_names.append(summit_name) 0639 0640 # For each collected branch name, check if there are some 0641 # branch templates for which the corresponding branch path 0642 # does not exit and (in case of explicit mapping) whether 0643 # all summit catalogs needed for scattering are available. 0644 # If this is the case, set missing paths for scattering. 0645 for branch_name in branch_names: 0646 0647 if ( branch_name in branch_templates 0648 and all([x in p.catalogs[SUMMIT_ID] for x in mapped_branch_names.get(branch.id, {}) 0649 .get(branch_name, [])]) 0650 ): 0651 # Assemble all branch catalog entries. 0652 for template in branch_templates[branch_name]: 0653 # Compose the branch catalog subdir and path. 0654 subdir = template[1] 0655 if branch.join_path: 0656 subpath = branch.join_path(branch_name, subdir, 0657 branch.by_lang) 0658 elif branch.by_lang: 0659 subpath = os.path.join(subdir, branch_name, 0660 branch.by_lang + ".po") 0661 else: 0662 subpath = os.path.join(subdir, 0663 branch_name + ".po") 0664 path = join_ncwd(branch.topdir, subpath) 0665 0666 # Skip this catalog if excluded from creation on 0667 # scatter, by filter on catalog name and subdir 0668 # (False -> excluded). 0669 scf = branch.scatter_create_filter 0670 if scf and not scf(branch_name, subdir): 0671 continue 0672 0673 # If not there already, add this path 0674 # to branch catalog entry, 0675 # and record later initialization from template. 0676 brcats = p.catalogs[branch.id].get(branch_name) 0677 if brcats is None: 0678 brcats = [] 0679 p.catalogs[branch.id][branch_name] = brcats 0680 if (path, subdir) not in brcats: 0681 brcats.append((path, subdir)) 0682 p.add_on_scatter[path] = template[0] 0683 0684 # In summit-over-dynamic-templates mode, 0685 # automatic vivification of summit catalogs must be active. 0686 if p.templates_dynamic: 0687 p.vivify_on_merge = True 0688 0689 # At merge in summit-over-templates mode, 0690 # if automatic vivification of summit catalogs requested, 0691 # add to the collection of summit catalogs any that should be created. 0692 p.add_on_merge = {} 0693 if ( p.over_templates and p.lang != p.templates_lang 0694 and "merge" in p.opmodes and (p.vivify_on_merge or options.create) 0695 ): 0696 # Collect all summit templates. 0697 if not p.templates_dynamic: 0698 summit_templates = collect_catalogs(p.summit.topdir_templates, 0699 ".pot", None, None, None, 0700 project, options) 0701 else: 0702 summit_templates = p.tproject.catalogs[SUMMIT_ID] 0703 0704 # Go through all summit templates, recording missing summit catalogs. 0705 for name, spec in summit_templates.items(): 0706 tpath, tsubdir = spec[0] # all summit catalogs unique 0707 if name not in p.catalogs[SUMMIT_ID]: 0708 # Compose the summit catalog path. 0709 spath = join_ncwd(p.summit.topdir, tsubdir, name + ".po") 0710 # Add this file to summit catalog entries. 0711 p.catalogs[SUMMIT_ID][name] = [(spath, tsubdir)] 0712 # Record later initialization from template. 0713 p.add_on_merge[spath] = tpath 0714 0715 # Convenient dictionary views of mappings. 0716 # - direct: branch_id->branch_name->summit_name 0717 # - part inverse: branch_id->summit_name->branch_name 0718 # - full inverse: summit_name->branch_id->branch_name 0719 p.direct_map = {} 0720 p.part_inverse_map = {} 0721 p.full_inverse_map = {} 0722 0723 # Initialize mappings by branch before the main loop for direct mappings, 0724 # because an explicit mapping may name a branch before it was processed 0725 # in the main loop. 0726 for branch_id in p.branch_ids: 0727 p.direct_map[branch_id] = {} 0728 for branch_name in p.catalogs[branch_id]: 0729 p.direct_map[branch_id][branch_name] = [] 0730 0731 # Add direct mappings. 0732 # - explicit 0733 for mapping in p.mappings: 0734 branch_id, branch_name = mapping[:2] 0735 if ( "gather" in p.opmodes 0736 and ( branch_id not in p.catalogs 0737 or branch_name not in p.catalogs[branch_id]) 0738 ): 0739 warning(_("@info", 0740 "No branch catalog corresponding to mapping %(mapping)s " 0741 "set by the summit configuration.", 0742 mapping=("('%s', '%s', ...)" % (branch_id, branch_name)))) 0743 continue 0744 summit_names = mapping[2:] 0745 p.direct_map[branch_id][branch_name] = summit_names 0746 # - implicit 0747 for branch_id in p.branch_ids: 0748 for branch_name in p.catalogs[branch_id]: 0749 if p.direct_map[branch_id][branch_name] == []: 0750 p.direct_map[branch_id][branch_name].append(branch_name) 0751 0752 # Convert subdir mappings into dictionary by branch ID and subdir. 0753 p.subdir_map = {} 0754 for bid, bsubdir, ssubdir in p.subdir_mappings: 0755 p.subdir_map[(bid, bsubdir)] = ssubdir 0756 0757 # Collect missing summit catalogs. 0758 needed_additions = [] 0759 for branch_id in p.branch_ids: 0760 for branch_name in p.catalogs[branch_id]: 0761 summit_names = p.direct_map[branch_id][branch_name] 0762 for summit_name in summit_names: 0763 if summit_name not in p.catalogs[SUMMIT_ID]: 0764 # Compose the path for the missing summit catalog. 0765 # Default the subdir to that of the current branch, 0766 # as it is the primary branch for this catalog. 0767 # Or use explicit subdir mapping if given. 0768 branch_path, branch_subdir = \ 0769 p.catalogs[branch_id][branch_name][0] 0770 dmkey = (branch_id, branch_subdir) 0771 summit_subdir = p.subdir_map.get(dmkey) or branch_subdir 0772 summit_path = join_ncwd(p.summit.topdir, summit_subdir, 0773 summit_name + catext) 0774 if "gather" in p.opmodes: 0775 if options.create: 0776 # Add summit catalog into list of existing catalogs; 0777 # it will be created for real on gather. 0778 p.catalogs[SUMMIT_ID][summit_name] = [ 0779 (summit_path, summit_subdir)] 0780 else: 0781 needed_additions.append((branch_path, summit_path)) 0782 elif "scatter" in p.opmodes: 0783 needed_additions.append((branch_path, summit_path)) 0784 0785 # Initialize inverse mappings. 0786 # - part inverse: 0787 for branch_id in p.branch_ids: 0788 p.part_inverse_map[branch_id] = {} 0789 for summit_name in p.catalogs[SUMMIT_ID]: 0790 p.part_inverse_map[branch_id][summit_name] = [] 0791 # - full inverse: 0792 for summit_name in p.catalogs[SUMMIT_ID]: 0793 p.full_inverse_map[summit_name] = {} 0794 for branch_id in p.branch_ids: 0795 p.full_inverse_map[summit_name][branch_id] = [] 0796 0797 # Add existing inverse mappings. 0798 for branch_id in p.branch_ids: 0799 for branch_name in sorted(p.catalogs[branch_id]): 0800 for summit_name in p.direct_map[branch_id][branch_name]: 0801 if summit_name in p.full_inverse_map: 0802 # - part inverse: 0803 pinv = p.part_inverse_map[branch_id][summit_name] 0804 if branch_name not in pinv: 0805 pinv.append(branch_name) 0806 # - full inverse: 0807 finv = p.full_inverse_map[summit_name][branch_id] 0808 if branch_name not in finv: 0809 finv.append(branch_name) 0810 0811 # Collect superfluous summit catalogs. 0812 needed_removals = [] 0813 for summit_name in p.catalogs[SUMMIT_ID]: 0814 src_branch_ids = [] 0815 for branch_id in project.branch_ids: 0816 if project.full_inverse_map[summit_name][branch_id]: 0817 src_branch_ids.append(branch_id) 0818 if not src_branch_ids: 0819 if "gather" in p.opmodes: 0820 if not options.create: 0821 summit_path = p.catalogs[SUMMIT_ID][summit_name][0][0] 0822 needed_removals.append(summit_path) 0823 0824 # Create function to assign precedence to a subdirectory. 0825 p.subdir_precedence = [os.path.normpath(sd) for sd in p.subdir_precedence] 0826 def calc_subdir_precedence (subdir): 0827 for i, test_subdir in enumerate(p.subdir_precedence): 0828 ltsd = len(test_subdir) 0829 if ( subdir.startswith(test_subdir) 0830 and subdir[ltsd:ltsd + 1] in ("", os.path.sep) 0831 ): 0832 return i 0833 return len(p.subdir_precedence) 0834 p.calc_subdir_precedence = calc_subdir_precedence 0835 0836 # Collect summit catalogs that should be moved. 0837 needed_moves = [] 0838 for summit_name in p.catalogs[SUMMIT_ID]: 0839 branch_subdirs = [] 0840 for branch_id in p.full_inverse_map[summit_name]: 0841 for branch_name in p.full_inverse_map[summit_name][branch_id]: 0842 branch_subdirs_1 = [] 0843 for bpath, bsubdir in p.catalogs[branch_id][branch_name]: 0844 dmkey = (branch_id, bsubdir) 0845 branch_subdirs_1.append(p.subdir_map.get(dmkey) or bsubdir) 0846 branch_subdirs.extend(branch_subdirs_1) 0847 if branch_subdirs: 0848 branch_subdirs = list(set(branch_subdirs)) 0849 subdir_precs = list(map(p.calc_subdir_precedence, branch_subdirs)) 0850 precs_subdirs = sorted(zip(subdir_precs, branch_subdirs)) 0851 branch_subdirs_sel = [sd for pr, sd in precs_subdirs 0852 if pr == precs_subdirs[0][0]] 0853 summit_subdir = p.catalogs[SUMMIT_ID][summit_name][0][1] 0854 if summit_subdir not in branch_subdirs_sel: 0855 summit_path = p.catalogs[SUMMIT_ID][summit_name][0][0] 0856 dpaths = [] 0857 for bsubdir in branch_subdirs_sel: 0858 dpath = join_ncwd(p.summit.topdir, bsubdir, 0859 summit_name + catext) 0860 dpaths.append(dpath) 0861 if "gather" in p.opmodes: 0862 if not options.create: 0863 needed_moves.append((summit_path, dpaths)) 0864 0865 # If catalog creation is not allowed, 0866 # complain about needed additions, removals, and moves. 0867 if needed_additions or needed_removals or needed_moves: 0868 if needed_additions: 0869 fmtlist = "\n".join("%s --> %s" % x 0870 for x in sorted(needed_additions)) 0871 warning(_("@info", 0872 "Some branch catalogs have no " 0873 "associated summit catalog " 0874 "(expected summit path given):\n" 0875 "%(filelist)s", 0876 filelist=fmtlist)) 0877 if needed_removals: 0878 fmtlist = "\n".join(sorted(needed_removals)) 0879 warning(_("@info", 0880 "Some summit catalogs have no " 0881 "associated branch catalogs:\n" 0882 "%(filelist)s", 0883 filelist=fmtlist)) 0884 if needed_moves: 0885 fmtlist = "\n".join("%s --| %s" % (x, " | ".join(y)) 0886 for x, y in sorted(needed_moves)) 0887 warning(_("@info", 0888 "Some summit catalogs should be " 0889 "moved to another subdirectory:\n" 0890 "%(filelist)s", 0891 filelist=fmtlist)) 0892 if "gather" in p.opmodes: 0893 error(_("@info", 0894 "Halting because catalog creation is not allowed " 0895 "(consider issuing %(opt)s option).", opt="--create")) 0896 0897 # Fill in defaults for missing fields in hook specs. 0898 for attr in p.__dict__: 0899 if attr.startswith("hook_"): 0900 p.__dict__[attr] = hook_fill_defaults(p.__dict__[attr]) 0901 0902 return p 0903 0904 0905 def split_path_in_project (project, path): 0906 0907 if os.path.isfile(path): 0908 if not path.endswith((".po", ".pot")): 0909 error(_("@info", 0910 "Non-PO file '%(file)s' given as catalog.", 0911 file=path)) 0912 0913 splits = [] 0914 for b in [project.summit] + project.branches: 0915 broot = os.path.abspath(b.topdir) 0916 apath = os.path.abspath(path) 0917 if apath.startswith(broot + os.path.sep) or apath == broot: 0918 subpath = apath[len(broot + os.path.sep):] 0919 # Split the path into catalog name and subdirectory. 0920 if os.path.isfile(apath): 0921 if b.split_path: 0922 catname, subdir = b.split_path(subpath) 0923 else: 0924 subdir = os.path.dirname(subpath) 0925 basename = os.path.basename(subpath) 0926 catname = basename[:basename.rfind(".")] 0927 if b.by_lang: 0928 # If this is by-language mode, 0929 # catalog path can be split only if of proper language, 0930 # and subdirectory and catalog name should backtrack. 0931 if catname != b.by_lang: 0932 continue 0933 catname = os.path.basename(subdir) 0934 subdir = os.path.dirname(subdir) 0935 elif os.path.isdir(apath): 0936 if b.split_path: 0937 catname = None 0938 dummy_subpath = os.path.join(subpath, "__dummy__.po") 0939 subdir = b.split_path(dummy_subpath)[1] 0940 else: 0941 subdir = subpath 0942 catname = None 0943 if b.by_lang: 0944 # If this is a leaf directory in by-language mode, 0945 # then actually a catalog has been selected, 0946 # and subdirectory and catalog name should backtrack. 0947 apath2 = os.path.join(subdir, b.by_lang + ".po") 0948 if os.path.isfile(apath2): 0949 catname = os.path.basename(subdir) 0950 subdir = os.path.dirname(subdir) 0951 # Collect the splitting. 0952 # Catalog name being None means that a subdirectory is selected, 0953 # and if subdirectory too is None, the whole branch is selected. 0954 if not catname and not subdir: 0955 subdir = None 0956 splits.append((b.id, subdir, catname)) 0957 if not splits: 0958 error(_("@info", 0959 "Path '%(path)s' is not covered by the summit configuration.", 0960 path=path)) 0961 0962 return splits 0963 0964 0965 def collect_partspecs (project, specargs): 0966 0967 partbids = [] 0968 partspecs = {} 0969 for specarg in specargs: 0970 # If the partial specification is a valid path, 0971 # convert it to operation target. 0972 optargets = [] 0973 if os.path.exists(specarg): 0974 splits = split_path_in_project(project, specarg) 0975 for bid, breldir, catname in splits: 0976 if catname: 0977 optarget = bid + ":" + catname 0978 elif breldir: 0979 optarget = bid + ":" + breldir + os.path.sep 0980 else: 0981 optarget = bid + ":" 0982 optargets.append(optarget) 0983 else: 0984 optargets = [specarg] 0985 0986 for optarget in optargets: 0987 lst = optarget.split(":", 1) 0988 if len(lst) < 2: 0989 fdname, = lst 0990 bid = None 0991 else: 0992 bid, fdname = lst 0993 if bid not in project.branch_ids and bid != SUMMIT_ID: 0994 error(_("@info", 0995 "Branch '%(branch)s' is not defined " 0996 "in the summit configuration.", 0997 branch=bid)) 0998 if bid and bid not in partbids: 0999 partbids.append(bid) 1000 if fdname: 1001 bsid = bid or SUMMIT_ID 1002 if bsid not in partspecs: 1003 partspecs[bsid] = [] 1004 partspecs[bsid].append(fdname) 1005 1006 return partspecs, partbids 1007 1008 1009 # Fill in defaults for missing fields in hook specs. 1010 def hook_fill_defaults (specs): 1011 1012 new_specs = [] 1013 for spec in specs: 1014 call = spec[0] 1015 branch_rx = r"" 1016 if len(spec) > 1: branch_rx = spec[1] 1017 name_rx = r"" 1018 if len(spec) > 2: name_rx = spec[2] 1019 new_specs.append((call, branch_rx, name_rx)) 1020 1021 return new_specs 1022 1023 1024 # Each catalog is represented by a dictionary entry: the key is the catalog 1025 # name, the value is the list of tuples of file path and subdirectory 1026 # relative to top (list in case there are several same-named catalogs in 1027 # different subdirectories). 1028 def collect_catalogs (topdir, catext, by_lang, ignored, split_path, 1029 project, options): 1030 1031 catalogs = {} 1032 topdir = os.path.normpath(topdir) 1033 for root, dirs, files in os.walk(topdir): 1034 for file in files: 1035 catn = "" 1036 if file.endswith(catext): 1037 if not by_lang: 1038 fpath = os.path.abspath(os.path.join(root, file)) 1039 if split_path: 1040 catn, spath = split_path(fpath[len(topdir) + 1:]) 1041 else: 1042 catn = file[0:file.rfind(".")] 1043 spath = root[len(topdir) + 1:] 1044 elif file == by_lang + ".po" or catext == ".pot": 1045 fpath = os.path.abspath(os.path.join(root, file)) 1046 if split_path: 1047 catn, spath = split_path(fpath[len(topdir) + 1:]) 1048 else: 1049 catn = os.path.basename(root) 1050 spath = os.path.dirname(root)[len(topdir) + 1:] 1051 1052 if catn: 1053 if not ignored or not ignored(fpath): 1054 if catn not in catalogs: 1055 catalogs[catn] = [] 1056 fpath = join_ncwd(fpath) 1057 spath = os.path.normpath(spath) 1058 catalogs[catn].append((fpath, spath)) 1059 1060 for catpaths in list(catalogs.values()): 1061 catpaths.sort(key=lambda x: x[0]) 1062 1063 return catalogs 1064 1065 1066 def summit_gather (project, options): 1067 1068 if ( project.over_templates and project.lang != project.templates_lang 1069 and not options.force): 1070 error(_("@info", 1071 "Gathering catalogs is normally not allowed " 1072 "in summit-over-static-templates mode. " 1073 "If this is the initial creation of summit catalogs, " 1074 "or externally injected branch catalogs need to be gathered, " 1075 "run with options %(opts)s.", 1076 opts="--create --force")) 1077 elif ( project.templates_dynamic 1078 and project.lang == project.templates_lang and not options.force): 1079 warning(_("@info", 1080 "Gathering templates is superfluous in " 1081 "summit-over-templates mode. " 1082 "If this is done to check whether gathering works, " 1083 "to supress this message run with option %(opt)s.", 1084 opt="--force")) 1085 1086 # Collect names of summit catalogs to gather. 1087 summit_names = select_summit_names(project, options) 1088 1089 # Setup progress indicator. 1090 upprog = lambda x=None: x 1091 if not options.verbose: 1092 catpaths = [project.catalogs[SUMMIT_ID][x][0][0] for x in summit_names] 1093 upprog = init_file_progress(catpaths, 1094 addfmt=t_("@info:progress", 1095 "Gathering: %(file)s")) 1096 1097 # Gather all selected catalogs. 1098 for name in summit_names: 1099 catpath = project.catalogs[SUMMIT_ID][name][0][0] 1100 if options.verbose: 1101 report(_("@info:progress", 1102 "Gathering %(file)s...", 1103 file=catpath)) 1104 upprogc = lambda: upprog(catpath) 1105 summit_gather_single(name, project, options, update_progress=upprogc) 1106 upprog() 1107 1108 1109 def summit_scatter (project, options): 1110 1111 if project.over_templates and project.lang == project.templates_lang: 1112 error(_("@info", 1113 "Scattering not possible on '%(lang)s' " 1114 "in summit-over-templates mode.", 1115 lang=project.templates_lang)) 1116 1117 scatter_specs = [] 1118 1119 # Select branches to scatter to. 1120 if not options.partbids or SUMMIT_ID in options.partbids: 1121 branch_ids = project.branch_ids 1122 else: 1123 branch_ids = options.partbids 1124 1125 # Collect catalogs to scatter through all selected branches. 1126 for branch_id in branch_ids: 1127 1128 branch_catalogs = select_branch_catalogs(branch_id, project, options) 1129 1130 for branch_name, branch_path, branch_subdir in branch_catalogs: 1131 1132 # Collect names of all the summit catalogs which this branch 1133 # catalog supplies messages to. 1134 summit_names = project.direct_map[branch_id][branch_name] 1135 1136 # Collect paths of selected summit catalogs. 1137 summit_paths = [] 1138 for summit_name in summit_names: 1139 if not summit_name in project.catalogs[SUMMIT_ID]: 1140 # Warning pertinent to this situation will have 1141 # been issued earlier, so just skip it here. 1142 #warning(_("@info", 1143 #"Missing summit catalog " 1144 #"for branch catalog '%(file)s'.", 1145 #file=branch_path)) 1146 continue 1147 summit_paths.append( 1148 project.catalogs[SUMMIT_ID][summit_name][0][0]) 1149 1150 # There may be no summit catalogs for this branch catalog. 1151 # The warning about this condition has been issued earlier, 1152 # just skip the branch catalog here. 1153 if summit_paths: 1154 scatter_specs.append((branch_id, branch_name, branch_subdir, 1155 branch_path, summit_paths)) 1156 1157 # Dummy entry to indicate branch switch. 1158 scatter_specs.append((branch_id, None, None, None, None)) 1159 1160 # Setup progress indicator. 1161 upprog = lambda x=None: x 1162 if not options.verbose: 1163 catpaths = [x[3] for x in scatter_specs if x[1]] 1164 upprog = init_file_progress(catpaths, 1165 addfmt=t_("@info:progress", 1166 "Scattering: %(file)s")) 1167 1168 # Scatter to branch catalogs. 1169 for scatter_spec in scatter_specs: 1170 branch_id, catpath = scatter_spec[0], scatter_spec[3] 1171 if catpath is not None: 1172 if options.verbose: 1173 report(_("@info:progress", 1174 "Scattering %(file)s...", 1175 file=catpath)) 1176 upprogc = lambda: upprog(catpath) 1177 summit_scatter_single(*(scatter_spec + (project, options, upprogc))) 1178 else: 1179 # Apply post-scatter hooks. 1180 if options.verbose: 1181 report(_("@info:progress", 1182 "Applying post-hook to branch %(branch)s...", 1183 branch=branch_id)) 1184 exec_hook_branch(branch_id, project.hook_on_scatter_branch) 1185 upprog() 1186 1187 1188 def summit_merge (project, options): 1189 1190 if project.over_templates and project.lang == project.templates_lang: 1191 error(_("@info", 1192 "Merging not possible on '%(lang)s' in " 1193 "summit-over-templates mode.", 1194 lang=project.templates_lang)) 1195 1196 merge_specs = [] 1197 1198 # Select branches to merge. 1199 if not options.partbids: 1200 branch_ids = project.branch_ids + [SUMMIT_ID] 1201 else: 1202 branch_ids = options.partbids 1203 1204 # Setup merging in summit. 1205 if SUMMIT_ID in branch_ids and project.summit.topdir_templates: 1206 branch_ids.remove(SUMMIT_ID) 1207 1208 # Collect names of summit catalogs to merge. 1209 summit_names = select_summit_names(project, options) 1210 1211 # Collect template catalogs to use. 1212 if not project.templates_dynamic: 1213 template_catalogs = collect_catalogs(project.summit.topdir_templates, 1214 ".pot", None, None, None, 1215 project, options) 1216 else: 1217 template_catalogs = project.tproject.catalogs[SUMMIT_ID] 1218 1219 # Collect data for summit catalogs to merge. 1220 for name in summit_names: 1221 summit_path, summit_subdir = project.catalogs[SUMMIT_ID][name][0] 1222 if name not in template_catalogs: 1223 warning(_("@info", 1224 "No template for summit catalog '%(file)s'.", 1225 file=summit_path)) 1226 continue 1227 template_path = template_catalogs[name][0][0] 1228 merge_specs.append((SUMMIT_ID, name, summit_subdir, 1229 summit_path, template_path, 1230 project.summit_wrapping, 1231 project.summit_fuzzy_merging)) 1232 1233 # Setup merging in branches. 1234 for branch_id in branch_ids: 1235 branch = project.bdict[branch_id] 1236 1237 # Skip branch if local merging not desired, or no templates defined. 1238 if (not branch.merge or branch.topdir_templates is None): 1239 continue 1240 1241 # Collect branch catalogs to merge. 1242 branch_catalogs = select_branch_catalogs(branch_id, project, options) 1243 1244 # Collect template catalogs to use. 1245 template_catalogs = collect_catalogs(branch.topdir_templates, ".pot", 1246 branch.by_lang, branch.ignored, 1247 branch.split_path, 1248 project, options) 1249 1250 # Collect data for branch catalogs to merge. 1251 for name, branch_path, branch_subdir in branch_catalogs: 1252 if not os.path.isfile(branch_path): 1253 # Catalog has been selected due to another operation mode, 1254 # which can create catalogs from scratch. 1255 continue 1256 if not name in template_catalogs: 1257 warning(_("@info", 1258 "No template for branch catalog '%(file)s'.", 1259 file=branch_path)) 1260 continue 1261 exact = False 1262 for template_path, template_subdir in template_catalogs[name]: 1263 if template_subdir == branch_subdir: 1264 exact = True 1265 break 1266 if not exact: 1267 warning(_("@info", 1268 "No exact template for branch catalog '%(file)s'.", 1269 file=branch_path)) 1270 continue 1271 merge_specs.append((branch_id, name, branch_subdir, 1272 branch_path, template_path, 1273 project.branches_wrapping, 1274 project.branches_fuzzy_merging)) 1275 1276 # Setup progress indicator. 1277 upprog = lambda x=None: x 1278 if not options.verbose: 1279 catpaths = [x[3] for x in merge_specs] 1280 upprog = init_file_progress(catpaths, 1281 addfmt=t_("@info:progress", 1282 "Merging: %(file)s")) 1283 1284 # Merge catalogs. 1285 for merge_spec in merge_specs: 1286 catpath = merge_spec[3] 1287 if options.verbose: 1288 report(_("@info:progress", 1289 "Merging %(file)s...", 1290 file=catpath)) 1291 upprogc = lambda: upprog(catpath) 1292 summit_merge_single(*(merge_spec + (project, options, upprogc))) 1293 upprog() 1294 1295 # Remove template tree in summit-over-dynamic-templates mode. 1296 if project.templates_dynamic: 1297 shutil.rmtree(project.tproject.summit.topdir) 1298 1299 1300 def summit_deps (project, options): 1301 1302 # Collect names of summit catalogs for which to report dependencies. 1303 summit_names = select_summit_names(project, options) 1304 1305 # Report dependencies for all selected catalogs. 1306 for summit_name in summit_names: 1307 if summit_name not in project.catalogs[SUMMIT_ID]: 1308 # May happen if there are some missing summit catalogs 1309 # to current branch catalogs, i.e. gather has not been made. 1310 continue 1311 summit_path = project.catalogs[SUMMIT_ID][summit_name][0][0] 1312 branch_paths = [] 1313 for branch_id in project.branch_ids: 1314 for branch_name in project.full_inverse_map[summit_name][branch_id]: 1315 for branch_path, d1 in project.catalogs[branch_id][branch_name]: 1316 branch_paths.append(branch_path) 1317 fmtbpaths = " ".join(branch_paths) 1318 if options.verbose: 1319 actype = _("@item:intext action performed on a catalog", 1320 "depends") 1321 report(": (%s) %s %s" % (actype, summit_path, fmtbpaths)) 1322 else: 1323 report(": %s %s" % (summit_path, fmtbpaths)) 1324 1325 1326 def select_branch_catalogs (branch_id, project, options): 1327 1328 # Shortcuts. 1329 pbcats = project.catalogs[branch_id] 1330 1331 # Select either all catalogs in this branch, 1332 # or those mentioned in the command line. 1333 if not options.partspecs: 1334 branch_catalogs = [] 1335 for name, spec in list(pbcats.items()): 1336 for path, subdir in spec: 1337 if options.selcatf(path): 1338 branch_catalogs.append((name, path, subdir)) 1339 else: 1340 # Select branch catalogs by command line specification. 1341 branch_catalogs = [] 1342 1343 # Process direct specifications (branch->summit). 1344 if branch_id in options.partspecs: 1345 for part_spec in options.partspecs[branch_id]: 1346 # If the catalog specification has path separators, 1347 # then it selects a complete subdir in the branch. 1348 branch_catalogs_l = [] 1349 if part_spec.find(os.sep) >= 0: 1350 sel_subdir = os.path.normpath(part_spec) 1351 one_found = False 1352 for name, spec in list(pbcats.items()): 1353 for path, subdir in spec: 1354 if sel_subdir == subdir: 1355 one_found = True 1356 if options.selcatf(path): 1357 branch_catalogs_l.append( 1358 (name, path, subdir)) 1359 if not one_found: 1360 error(_("@info", 1361 "No catalogs in subdirectory '%(dir)s' " 1362 "of branch '%(branch)s'.", 1363 dir=sel_subdir, branch=branch_id)) 1364 else: 1365 # Otherwise, specific catalog is selected. 1366 sel_name = part_spec 1367 one_found = False 1368 for name, spec in list(pbcats.items()): 1369 if sel_name == name: 1370 for path, subdir in spec: 1371 one_found = True 1372 if options.selcatf(path): 1373 branch_catalogs_l.append( 1374 (name, path, subdir)) 1375 break 1376 if not one_found: 1377 error(_("@info", 1378 "No catalog named '%(name)s' " 1379 "in branch '%(branch)s'.", 1380 name=sel_name, branch=branch_id)) 1381 1382 # Also select all branch catalogs which contribute to same 1383 # summit catalogs as the already selected ones. 1384 branch_catalogs_l2 = [] 1385 dmap = project.direct_map[branch_id] 1386 pimap = project.part_inverse_map[branch_id] 1387 for branch_name, d1, d2 in branch_catalogs_l: 1388 if branch_name in dmap: 1389 for summit_name in dmap[branch_name]: 1390 if summit_name in pimap: 1391 for name in pimap[summit_name]: 1392 for path, subdir in pbcats[name]: 1393 if options.selcatf(path): 1394 branch_catalogs_l2.append( 1395 (name, path, subdir)) 1396 1397 branch_catalogs.extend(branch_catalogs_l) 1398 branch_catalogs.extend(branch_catalogs_l2) 1399 1400 # Process inverse specifications (summit->branch). 1401 if SUMMIT_ID in options.partspecs: 1402 for part_spec in options.partspecs[SUMMIT_ID]: 1403 if part_spec.find(os.sep) >= 0: 1404 # Complete subdir. 1405 sel_subdir = os.path.normpath(part_spec) 1406 cats = [] 1407 for name, spec in list(project.catalogs[SUMMIT_ID].items()): 1408 path, subdir = spec[0] # all summit catalogs unique 1409 if sel_subdir == subdir: 1410 bnames = project.full_inverse_map[name][branch_id] 1411 for bname in bnames: 1412 if bname in pbcats: 1413 for bpath, bsubdir in pbcats[bname]: 1414 if options.selcatf(bpath): 1415 cats.append((bname, bpath, bsubdir)) 1416 branch_catalogs.extend(cats) 1417 else: 1418 # Specific catalog. 1419 sel_name = part_spec 1420 if not sel_name in project.catalogs[SUMMIT_ID]: 1421 error(_("@info", 1422 "No summit catalog named '%(name)s'.", 1423 name=sel_name)) 1424 bnames = project.full_inverse_map[sel_name][branch_id] 1425 for bname in bnames: 1426 if bname in pbcats: 1427 for bpath, bsubdir in pbcats[bname]: 1428 if options.selcatf(bpath): 1429 branch_catalogs.append( 1430 (bname, bpath, bsubdir)) 1431 1432 # Same catalogs may have been selected multiple times, remove. 1433 branch_catalogs = list(set(branch_catalogs)) 1434 1435 # Sort by path. 1436 branch_catalogs.sort(key=lambda x: x[1]) 1437 # ...sorting is not only for looks, but to establish priority of 1438 # supplying comments to summit messages. 1439 1440 return branch_catalogs 1441 1442 1443 def select_summit_names (project, options): 1444 1445 # Collect all summit catalogs selected explicitly or implicitly. 1446 summit_names = [] 1447 if options.partspecs is None: 1448 for name, spec in list(project.catalogs[SUMMIT_ID].items()): 1449 path, subdir = spec[0] # summit catalogs are unique 1450 if options.selcatf(path): 1451 summit_names.append(name) 1452 else: 1453 for branch_id in options.partspecs: 1454 for part_spec in options.partspecs[branch_id]: 1455 1456 if branch_id == SUMMIT_ID: # explicit by summit reference 1457 if part_spec.find(os.sep) >= 0: # whole subdir 1458 sel_subdir = os.path.normpath(part_spec) 1459 one_found = False 1460 for name, spec in list(project.catalogs[SUMMIT_ID].items()): 1461 path, subdir = spec[0] # summit catalogs are unique 1462 if sel_subdir == subdir: 1463 one_found = True 1464 if options.selcatf(path): 1465 summit_names.append(name) 1466 if not one_found: 1467 error(_("@info", 1468 "No summit directory named '%(name)s'.", 1469 name=sel_subdir)) 1470 else: # single name 1471 sel_name = part_spec 1472 spec = project.catalogs[SUMMIT_ID].get(sel_name) 1473 if not spec: 1474 error(_("@info", 1475 "No summit catalog named '%(name)s'.", 1476 name=sel_name)) 1477 path, subdir = spec[0] # summit catalogs are unique 1478 if options.selcatf(path): 1479 summit_names.append(sel_name) 1480 1481 else: # implicit by branch reference 1482 if part_spec.find(os.sep) >= 0: # whole subdir 1483 sel_subdir = os.path.normpath(part_spec) 1484 one_found = False 1485 for name, spec in list(project.catalogs[branch_id].items()): 1486 for path, subdir in spec: 1487 if sel_subdir == subdir: 1488 one_found = True 1489 if options.selcatf(path): 1490 summit_names.extend( 1491 project.direct_map[branch_id][name]) 1492 break 1493 if not one_found: 1494 error(_("@info", 1495 "No directory named '%(name)s' " 1496 "in branch '%(branch)s'.", 1497 name=sel_subdir, branch=branch_id)) 1498 else: # single name 1499 sel_name = part_spec 1500 spec = project.catalogs[branch_id].get(sel_name) 1501 if not spec: 1502 error(_("@info", 1503 "No catalog named '%(name)s' " 1504 "in branch '%(branch)s'.", 1505 name=sel_name, branch=branch_id)) 1506 for path, subdir in spec: 1507 if options.selcatf(path): 1508 summit_names.extend( 1509 project.direct_map[branch_id][sel_name]) 1510 break 1511 1512 # Make names unique and sort by path. 1513 summit_names = list(set(summit_names)) 1514 summit_names.sort(key=lambda x: project.catalogs[SUMMIT_ID].get(x, [[""]])[0][0]) 1515 1516 # Additionaly sort by subdirectory precedence. 1517 # This is necessary so that catalogs can be properly moved when gathering, 1518 # in case a higher precedence subdirectory was not created before. 1519 # Default "~" means that catalogs with no paths will be sorted at end. 1520 summit_names.sort(key=lambda x: project.calc_subdir_precedence( 1521 project.catalogs[SUMMIT_ID].get(x, [["", "~"]])[0][1])) 1522 1523 return summit_names 1524 1525 1526 def summit_gather_single (summit_name, project, options, 1527 phony=False, pre_summit_names=(), memo_store=None, 1528 update_progress=(lambda: None)): 1529 1530 if memo_store is not None: 1531 memo_key = (summit_name, tuple(sorted(pre_summit_names))) 1532 if memo_key in memo_store: # value can be None 1533 return memo_store.get(memo_key) 1534 1535 update_progress() 1536 1537 summit_path = project.catalogs[SUMMIT_ID][summit_name][0][0] 1538 summit_subdir = project.catalogs[SUMMIT_ID][summit_name][0][1] 1539 1540 update_from_old = ( os.path.exists(summit_path) 1541 and not project.templates_dynamic) 1542 1543 # Do not overwrite the old summit catalog here if it exists, 1544 # as it will be needed for comparison later. 1545 monitored = update_from_old 1546 summit_cat = Catalog("", monitored=monitored, 1547 wrapping=project.summit_wrapping, create=True) 1548 summit_cat.filename = summit_path 1549 1550 # Collect branches in which this summit catalog has corresponding 1551 # branch catalogs, in order of branch priority. 1552 src_branch_ids = [] 1553 for branch_id in project.branch_ids: 1554 if project.full_inverse_map[summit_name][branch_id]: 1555 src_branch_ids.append(branch_id) 1556 1557 # If there are no branch catalogs, 1558 # then the current summit catalog is to be removed. 1559 if not src_branch_ids: 1560 if phony: # cannot happen 1561 error(_("@info", 1562 "Phony gather on summit catalog which is to be removed.")) 1563 1564 # Remove by version control, if any. 1565 if project.summit_vcs: 1566 if not project.summit_vcs.remove(summit_path): 1567 warning(_("@info", 1568 "Cannot remove '%(path)s' from version control.", 1569 path=summit_path)) 1570 # If not removed by version control, plainly delete. 1571 if os.path.isfile(summit_path): 1572 os.unlink(summit_path) 1573 if os.path.isfile(summit_path): 1574 warning(_("@info", 1575 "Cannot remove '%(path)s' from disk.", 1576 path=summit_path)) 1577 1578 if not os.path.isfile(summit_path): 1579 if options.verbose: 1580 actype = _("@item:intext action performed on a catalog", 1581 "gathered-removed") 1582 report("- (%s) %s" % (actype, summit_path)) 1583 elif not options.quiet: 1584 report("- %s" % summit_path) 1585 1586 # Skip the rest, nothing to gather. 1587 if memo_store is not None: 1588 memo_store[memo_key] = summit_cat 1589 return summit_cat 1590 1591 # Open all corresponding branch catalogs. 1592 # For each branch catalog, also phony-gather any dependent summit 1593 # catalogs. Phony means not to take into account branch catalogs which 1594 # map to current summit catalog if it is higher in their queue than 1595 # the phony-gathered one, and not to sync phony-gathered catalog; 1596 # this is needed in order that any new messages get inserted 1597 # uniquely and deterministically in case of split-mappings. 1598 bcat_pscats = {} 1599 if phony or memo_store is not None: 1600 sub_memo_store = memo_store 1601 else: 1602 sub_memo_store = {} 1603 for branch_id in src_branch_ids: 1604 1605 branch = project.bdict[branch_id] 1606 if isinstance(branch.insert_nosim, (list, tuple)): 1607 apply_insert_nosim = lambda sn, sd: ( 1608 any(re.search(rs, sn) for rs in branch.insert_nosim)) 1609 elif callable(branch.insert_nosim): 1610 apply_insert_nosim = lambda sn, sd: branch.insert_nosim(sn, sd) 1611 else: 1612 apply_insert_nosim = lambda sn, sd: bool(branch.insert_nosim) 1613 1614 bcat_pscats[branch_id] = [] 1615 for branch_name in project.full_inverse_map[summit_name][branch_id]: 1616 1617 # In phony-gather, do not use branch catalogs with split-mappings 1618 # which map to one of the summit catalogs among previous. 1619 phony_skip = False 1620 for dep_summit_name in project.direct_map[branch_id][branch_name]: 1621 if dep_summit_name in pre_summit_names: 1622 phony_skip = True 1623 break 1624 if phony_skip: 1625 continue 1626 1627 # Gather and open dependent summit catalogs. 1628 dep_summit_cats = [] 1629 sub_pre_summit_names = list(pre_summit_names) 1630 for dep_summit_name in project.direct_map[branch_id][branch_name]: 1631 if dep_summit_name == summit_name: 1632 sub_pre_summit_names.append(summit_name) 1633 continue 1634 dep_summit_cat = summit_gather_single(dep_summit_name, 1635 project, options, 1636 True, 1637 sub_pre_summit_names, 1638 sub_memo_store, 1639 update_progress) 1640 if dep_summit_cat is not None: 1641 dep_summit_cats.append(dep_summit_cat) 1642 1643 # Open all branch catalogs of this name, ordered by path, 1644 # link them to the same dependent summit catalogs. 1645 for path, subdir in project.catalogs[branch_id][branch_name]: 1646 update_progress() 1647 1648 # Apply hooks to branch catalog file, creating temporaries. 1649 tmp_path = None 1650 if project.hook_on_gather_file_branch: 1651 # Temporary path should be such as to not modify the 1652 # catalog name (e.g. appending ".mod" could make ".po" 1653 # a part of the name). 1654 tmp_path = path + "~mod" 1655 shutil.copyfile(path, tmp_path) 1656 exec_hook_file(branch_id, branch_name, subdir, tmp_path, 1657 project.hook_on_gather_file_branch) 1658 1659 branch_cat = Catalog(tmp_path or path, monitored=False) 1660 if tmp_path: # as soon as catalog is opened, no longer needed 1661 os.unlink(tmp_path) 1662 1663 # Apply hooks to branch catalog. 1664 if project.hook_on_gather_cat_branch: 1665 exec_hook_cat(branch_id, branch_name, subdir, branch_cat, 1666 project.hook_on_gather_cat_branch) 1667 branch_cat.sync_map() 1668 1669 # Apply hooks to all branch catalog messages here, 1670 # as they may modify message keys. 1671 if project.hook_on_gather_msg_branch: 1672 for msg in branch_cat: 1673 update_progress() 1674 exec_hook_msg(branch_id, branch_name, subdir, 1675 msg, branch_cat, 1676 project.hook_on_gather_msg_branch) 1677 branch_cat.sync_map() 1678 1679 insert_nosim = apply_insert_nosim(branch_name, subdir) 1680 1681 bcat_pscats[branch_id].append((branch_cat, dep_summit_cats, 1682 insert_nosim)) 1683 1684 # On phony gather, in case of split mappings, 1685 # it may happen that there are no corresponding branch catalogs. 1686 if phony and not any(bcat_pscats.values()): 1687 if memo_store is not None: 1688 memo_store[memo_key] = None 1689 return None 1690 1691 # Select primary branch catalog. 1692 prim_branch_cat = None 1693 for branch_id in src_branch_ids: 1694 if bcat_pscats[branch_id]: 1695 prim_branch_cat = bcat_pscats[branch_id][0][0] 1696 break 1697 assert prim_branch_cat is not None 1698 1699 # Gather messages through branch catalogs. 1700 for branch_id in src_branch_ids: 1701 for branch_cat, dep_summit_cats, insert_nosim in bcat_pscats[branch_id]: 1702 is_primary = branch_cat is prim_branch_cat 1703 summit_gather_single_bcat(branch_id, branch_cat, is_primary, 1704 summit_cat, monitored, dep_summit_cats, 1705 insert_nosim, 1706 project, options, update_progress) 1707 1708 # Gather the summit header according to primary branch. 1709 summit_gather_single_header(summit_cat, prim_branch_cat, project, options) 1710 1711 # Apply hooks to the summit messages. 1712 if project.hook_on_gather_msg: 1713 for msg in summit_cat: 1714 exec_hook_msg(SUMMIT_ID, summit_cat.name, summit_subdir, 1715 msg, summit_cat, project.hook_on_gather_msg) 1716 1717 # Apply hooks to the summit catalog. 1718 exec_hook_cat(SUMMIT_ID, summit_cat.name, summit_subdir, summit_cat, 1719 project.hook_on_gather_cat) 1720 1721 # If phony-gather, stop here and return summit catalog for reference. 1722 if phony: 1723 if memo_store is not None: 1724 memo_store[memo_key] = summit_cat 1725 return summit_cat 1726 1727 # If the old summit catalog exists, compare with the new. 1728 # If there were any modified entries, or their order changed, 1729 # replace the old with the new summit catalog. 1730 # Copy over unmodified entries from the old catalog, 1731 # to avoid line reformatting. 1732 if update_from_old: 1733 old_cat = Catalog(summit_path, monitored=monitored, 1734 wrapping=project.summit_wrapping) 1735 summit_created = False 1736 replace = False 1737 # Compare headers without some insignificant fields. 1738 if cmpnorm_hdr(summit_cat.header) == cmpnorm_hdr(old_cat.header): 1739 summit_cat.header = old_cat.header 1740 else: 1741 replace = True 1742 # Compare messages and their positions. 1743 for pos in range(len(summit_cat)): 1744 update_progress() 1745 old_pos = old_cat.find(summit_cat[pos]) 1746 if pos != old_pos: 1747 replace = True 1748 if old_pos >= 0: 1749 if summit_cat[pos] == old_cat[old_pos]: 1750 summit_cat[pos] = old_cat[old_pos] 1751 else: 1752 replace = True 1753 # Compare lengths. 1754 if len(summit_cat) != len(old_cat): 1755 replace = True 1756 else: 1757 summit_created = True 1758 replace = True 1759 1760 # Check if the catalog needs to be moved to another subdirectory. 1761 branch_subdirs = [] 1762 for branch_id in project.full_inverse_map[summit_name]: 1763 for branch_name in project.full_inverse_map[summit_name][branch_id]: 1764 branch_subdirs_1 = [] 1765 for bpath, bsubdir in project.catalogs[branch_id][branch_name]: 1766 bsubdir = project.subdir_map.get((branch_id, bsubdir), bsubdir) 1767 branch_subdirs_1.append(bsubdir) 1768 branch_subdirs_1.sort() 1769 branch_subdirs.extend(branch_subdirs_1) 1770 new_summit_path = summit_path 1771 if branch_subdirs: 1772 branch_subdirs = list(set(branch_subdirs)) 1773 subdir_precs = list(map(project.calc_subdir_precedence, branch_subdirs)) 1774 precs_subdirs = sorted(zip(subdir_precs, branch_subdirs)) 1775 branch_subdirs_sel = [sd for pr, sd in precs_subdirs 1776 if pr == precs_subdirs[0][0]] 1777 if summit_subdir not in branch_subdirs_sel: 1778 catext = summit_path[summit_path.rfind("."):] 1779 new_summit_path = join_ncwd(project.summit.topdir, 1780 branch_subdirs_sel[0], 1781 summit_name + catext) 1782 1783 if replace or summit_cat.filename != new_summit_path: 1784 added = False 1785 moved = False 1786 1787 if replace: 1788 # Set template creation date for the summit catalog 1789 # to the current date. 1790 # Do not try to trust branch template creation dates, 1791 # e.g. by copying the latest one. 1792 summit_cat.header.set_field("POT-Creation-Date", format_datetime(), 1793 before="PO-Revision-Date", 1794 reorder=True) 1795 1796 # Sync to disk. 1797 summit_cat.sync() 1798 1799 # Apply hooks to summit catalog file. 1800 exec_hook_file(SUMMIT_ID, summit_cat.name, summit_subdir, 1801 summit_cat.filename, project.hook_on_gather_file) 1802 1803 if summit_created: 1804 added = True 1805 # Add to version control. 1806 if ( project.summit_vcs 1807 and not project.summit_vcs.is_versioned(summit_cat.filename) 1808 ): 1809 if not project.summit_vcs.add(summit_cat.filename): 1810 warning(_("@info", 1811 "Cannot add '%(file)s' to version control.", 1812 file=summit_cat.filename)) 1813 else: 1814 added = True 1815 1816 if summit_cat.filename != new_summit_path: 1817 if project.summit_vcs: 1818 if not project.summit_vcs.move(summit_cat.filename, 1819 new_summit_path): 1820 warning(_("@info", 1821 "Cannot move '%(srcfile)s' to '%(dstfile)s'.", 1822 srcfile=summit_cat.filename, 1823 dstfile=new_summit_path)) 1824 else: 1825 summit_cat.filename = new_summit_path 1826 moved = True 1827 1828 branch_paths = [] 1829 for branch_id in src_branch_ids: 1830 for branch_cat, dep_summit_cats, insert_nosim in bcat_pscats[branch_id]: 1831 branch_paths.append(branch_cat.filename) 1832 paths_str = " ".join(branch_paths) 1833 if options.verbose: 1834 if added: 1835 actype = _("@item:intext action performed on a catalog", 1836 "gathered-added") 1837 report(">+ (%s) %s %s" 1838 % (actype, summit_cat.filename, paths_str)) 1839 elif moved: 1840 actype = _("@item:intext action performed on a catalog", 1841 "gathered-moved") 1842 report(">| (%s) %s %s" 1843 % (actype, summit_cat.filename, paths_str)) 1844 else: 1845 actype = _("@item:intext action performed on a catalog", 1846 "gathered") 1847 report("> (%s) %s %s" 1848 % (actype, summit_cat.filename, paths_str)) 1849 elif not options.quiet: 1850 if added: 1851 report(">+ %s %s" % (summit_cat.filename, paths_str)) 1852 elif moved: 1853 report(">| %s %s" % (summit_cat.filename, paths_str)) 1854 else: 1855 report("> %s %s" % (summit_cat.filename, paths_str)) 1856 1857 if memo_store is not None: 1858 memo_store[memo_key] = summit_cat 1859 return summit_cat 1860 1861 1862 def cmpnorm_hdr (hdr): 1863 1864 rhdr = Header(hdr) 1865 for field in ( 1866 "POT-Creation-Date", 1867 ): 1868 rhdr.remove_field(field) 1869 return rhdr 1870 1871 1872 def extkey_msg (msg): 1873 1874 # NOTE: If computation of context pad is modified, 1875 # padded messages in existing summit catalogs will get fuzzy 1876 # on next merge with newly gathered templates. 1877 1878 msg = MessageUnsafe(msg) 1879 if msg.msgid_plural is not None: 1880 h = hashlib.md5() 1881 h.update(msg.msgid_plural.encode("UTF-8")) 1882 ctxtpad = h.hexdigest() 1883 else: 1884 # Something that looks like a hex digest but slightly shorter, 1885 # so that it does not match any real digest. 1886 ctxtpad = "abcd1234efgh5665hgfe4321dcba" 1887 msg.auto_comment.append("%s msgctxt-pad %s" 1888 % (_summit_tag_kwprop, ctxtpad)) 1889 if msg.msgctxt is None: 1890 msg.msgctxt = "%s" % ctxtpad 1891 else: 1892 msg.msgctxt = "%s|%s" % (msg.msgctxt, ctxtpad) 1893 1894 return msg 1895 1896 1897 def summit_gather_single_bcat (branch_id, branch_cat, is_primary, 1898 summit_cat, monitored, dep_summit_cats, 1899 insert_nosim, 1900 project, options, update_progress): 1901 1902 MessageType = (Message if monitored else MessageUnsafe) 1903 1904 # Go through messages in the branch catalog, merging them with 1905 # existing summit messages, or collecting for later insertion. 1906 # Do not insert new messages immediately, as source references may be 1907 # updated by merging, which reflects on heuristic insertion. 1908 # Ignore messages present in dependent summit catalogs. 1909 msgs_to_merge = [] 1910 msgs_to_insert = [] 1911 xkpairs = [] 1912 for msg in branch_cat: 1913 update_progress() 1914 1915 # Do not gather obsolete messages. 1916 if msg.obsolete: 1917 continue 1918 1919 # Normalizations when gathering templates, 1920 # in case extraction tool needs to have its sanity checked, 1921 # or certain language files stand in for true templates. 1922 if project.lang == project.templates_lang: 1923 msg.manual_comment[:] = [] 1924 msg.unfuzzy() 1925 if msg.msgid_plural is None: 1926 msg.msgstr[:] = [""] 1927 else: 1928 msg.msgstr[:] = ["", ""] 1929 1930 # Construct branch message with extended key. 1931 xkmsg = extkey_msg(msg) 1932 1933 # Do not gather messages belonging to depending summit catalogs. 1934 in_dep = False 1935 for dep_summit_cat in dep_summit_cats: 1936 if msg in dep_summit_cat or xkmsg in dep_summit_cat: 1937 in_dep = True 1938 break 1939 if in_dep: 1940 continue 1941 1942 # If the summit message for the original branch message exists, 1943 # but their extended keys do not match, 1944 # switch to branch message with extended key. 1945 summit_msg = summit_cat.get(msg) 1946 if summit_msg and extkey_msg(summit_msg).key != xkmsg.key: 1947 xkpairs.append((msg, xkmsg)) 1948 msg = xkmsg 1949 summit_msg = summit_cat.get(msg) 1950 1951 # Collect the branch message for merging or insertion. 1952 if summit_msg is not None: 1953 msgs_to_merge.append((msg, summit_msg)) 1954 else: 1955 msgs_to_insert.append(msg) 1956 1957 # If some messages had to have extended keys, update branch catalog. 1958 if xkpairs: 1959 for msg, xkmsg in xkpairs: 1960 branch_cat.remove_on_sync(msg) 1961 branch_cat.add_last(xkmsg) 1962 branch_cat.sync_map() 1963 1964 # Merge messages already in the summit catalog. 1965 if msgs_to_merge: 1966 for msg, summit_msg in msgs_to_merge: 1967 # Merge the message. 1968 gather_merge_msg(summit_msg, msg) 1969 # Update automatic comments. 1970 summit_override_auto(summit_msg, msg, branch_id, is_primary) 1971 # Equip any new summit tags to the merged message. 1972 summit_set_tags(summit_msg, branch_id, project) 1973 1974 # Insert messages not already in the summit catalog. 1975 if msgs_to_insert: 1976 # Pair messages to insert from branch with summit messages 1977 # having common source files. 1978 # If summit is empty, this is primary branch catalog, so make 1979 # only one dummy pair to preserve original ordering of messages. 1980 summit_msgs_by_src_dict = dict(summit_cat.messages_by_source()) 1981 if summit_msgs_by_src_dict: 1982 msgs_by_src = branch_cat.messages_by_source() 1983 else: 1984 msgs_by_src = [("", branch_cat)] 1985 1986 # Collect possible source file synonyms to those in the summit catalog. 1987 fnsyn = branch_cat.detect_renamed_sources(summit_cat) 1988 1989 # Prepare messages for insertion into summit. 1990 summit_msg_by_msg = {} 1991 for msg in msgs_to_insert: 1992 update_progress() 1993 summit_msg = MessageType(msg) 1994 summit_set_tags(summit_msg, branch_id, project) 1995 summit_msg_by_msg[msg] = summit_msg 1996 1997 # Insert branch messages into summit source by source. 1998 for src, msgs in msgs_by_src: 1999 2000 # Assemble collection of summit messages from same source file. 2001 summit_msgs = [] 2002 for osrc in [src] + fnsyn.get(src, []): 2003 summit_msgs.extend(summit_msgs_by_src_dict.get(osrc, [])) 2004 2005 # If existing summit messages from same source found, 2006 # insert branch messages around those summit messages. 2007 # Otherwise, just append them at the end. 2008 if summit_msgs: 2009 2010 # Assemble groups of messages by same msgid and same msgctxt, 2011 # for insertion by similarity. 2012 if not insert_nosim: 2013 smsgs_by_msgid = {} 2014 smsgs_by_msgctxt = {} 2015 for smsg in summit_msgs: 2016 if smsg.msgid not in smsgs_by_msgid: 2017 smsgs_by_msgid[smsg.msgid] = [] 2018 smsgs_by_msgid[smsg.msgid].append(smsg) 2019 if smsg.msgctxt is not None: 2020 if smsg.msgctxt not in smsgs_by_msgctxt: 2021 smsgs_by_msgctxt[smsg.msgctxt] = [] 2022 smsgs_by_msgctxt[smsg.msgctxt].append(smsg) 2023 2024 insertions = [] 2025 for msg in msgs: 2026 update_progress() 2027 new_summit_msg = summit_msg_by_msg.get(msg) 2028 if new_summit_msg is None: 2029 continue 2030 2031 # Existing summit message to where (after or before) 2032 # current message is to be inserted. 2033 summit_msg_ref = None 2034 before = False 2035 2036 # Try to insert message by similarity. 2037 # Similarity is checked by groups, 2038 # such that for each group there is a message part 2039 # which is compared for similarity. 2040 if not insert_nosim: 2041 for summit_msgs_group, matt, forceins in ( 2042 (smsgs_by_msgid.get(msg.msgid), "msgctxt", True), 2043 (smsgs_by_msgctxt.get(msg.msgctxt), "msgid", True), 2044 (summit_msgs, "key", False), 2045 ): 2046 if not summit_msgs_group: 2047 continue 2048 2049 # Shortcut: if only one summit message in the group 2050 # and insertion forced, insert after it. 2051 if len(summit_msgs_group) == 1 and forceins: 2052 summit_msg_ref = summit_msgs_group[-1] 2053 break 2054 2055 # Does the message have the part to be matched? 2056 mval = msg.get(matt) 2057 if mval is None: 2058 continue 2059 2060 # Find existing message with the most similar 2061 # matching attribute. 2062 seqm = SequenceMatcher(None, mval, "") 2063 maxr = 0.0 2064 for summit_msg in summit_msgs_group: 2065 smval = summit_msg.get(matt) 2066 if smval is None: 2067 continue 2068 seqm.set_seq2(smval) 2069 r = seqm.ratio() 2070 if maxr <= r: 2071 maxr = r 2072 maxr_summit_msg = summit_msg 2073 2074 # If similar enough message has been found, 2075 # set insertion position after it. 2076 # Otherwise, insert after last summit message 2077 # in the group if insertion forced. 2078 if maxr > 0.6: 2079 summit_msg_ref = maxr_summit_msg 2080 break 2081 elif forceins: 2082 summit_msg_ref = summit_msgs_group[-1] 2083 break 2084 2085 # If no similar existing message, set position before 2086 # the summit message with first greater source reference 2087 # line number, if any such. 2088 if summit_msg_ref is None and src: 2089 for summit_msg in summit_msgs: 2090 if msg.source[0][1] < summit_msg.source[0][1]: 2091 summit_msg_ref = summit_msg 2092 before = True 2093 break 2094 2095 # If not insertion by source references, insert last. 2096 if summit_msg_ref is None: 2097 summit_msg_ref = summit_msgs[-1] 2098 2099 # Record insertion. 2100 pos = summit_cat.find(summit_msg_ref) 2101 if not before: 2102 pos += 1 2103 insertions.append((new_summit_msg, pos)) 2104 2105 # Insert ordered messages into catalog. 2106 summit_cat.add_more(insertions) 2107 2108 else: 2109 for msg in msgs: 2110 update_progress() 2111 new_summit_msg = summit_msg_by_msg.get(msg) 2112 if new_summit_msg is not None: 2113 summit_cat.add_last(new_summit_msg) 2114 2115 2116 def gather_merge_msg (summit_msg, msg): 2117 2118 if summit_msg.key != msg.key: 2119 error(_("@info", 2120 "Cannot gather messages with different keys.")) 2121 if (summit_msg.msgid_plural is None) != (msg.msgid_plural is None): 2122 error(_("@info", 2123 "Cannot gather messages with different plurality.")) 2124 2125 if ( (summit_msg.translated and msg.translated) 2126 or (summit_msg.fuzzy and msg.fuzzy) 2127 or (summit_msg.untranslated and msg.untranslated) 2128 ): 2129 if not summit_msg.manual_comment: 2130 summit_msg.manual_comment = Monlist(msg.manual_comment) 2131 if msg.msgid_plural is not None: 2132 summit_msg.msgid_plural = msg.msgid_plural 2133 summit_msg.msgstr = Monlist(msg.msgstr) 2134 2135 elif summit_msg.fuzzy and msg.translated: 2136 summit_msg.manual_comment = Monlist(msg.manual_comment) 2137 if summit_msg.msgid_plural is None or msg.msgid_plural is not None: 2138 if msg.msgid_plural is not None: 2139 summit_msg.msgid_plural = msg.msgid_plural 2140 summit_msg.msgstr = Monlist(msg.msgstr) 2141 if summit_msg.msgid_plural == msg.msgid_plural: 2142 summit_msg.unfuzzy() 2143 2144 elif summit_msg.untranslated and (msg.translated or msg.fuzzy): 2145 summit_msg.manual_comment = Monlist(msg.manual_comment) 2146 if summit_msg.msgid_plural is None or msg.msgid_plural is not None: 2147 if msg.fuzzy: 2148 summit_msg.msgctxt_previous = msg.msgctxt_previous 2149 summit_msg.msgid_previous = msg.msgid_previous 2150 summit_msg.msgid_plural_previous = msg.msgid_plural_previous 2151 if msg.msgid_plural is not None: 2152 summit_msg.msgid_plural = msg.msgid_plural 2153 summit_msg.msgstr = Monlist(msg.msgstr) 2154 summit_msg.fuzzy = msg.fuzzy 2155 2156 2157 def summit_gather_single_header (summit_cat, prim_branch_cat, 2158 project, options): 2159 2160 # Copy over comments from the primary branch catalog. 2161 hdr = summit_cat.header 2162 bhdr = prim_branch_cat.header 2163 hdr.title = bhdr.title 2164 hdr.copyright = bhdr.copyright 2165 hdr.license = bhdr.license 2166 hdr.author = bhdr.author 2167 hdr.comment = bhdr.comment 2168 2169 # Copy over standard fields from the primary branch catalog. 2170 for fname in [x[0] for x in Header().field]: 2171 fvalue = prim_branch_cat.header.get_field_value(fname) 2172 if fvalue is not None: 2173 summit_cat.header.set_field(fname, fvalue) 2174 else: 2175 summit_cat.header.remove_field(fname) 2176 2177 # Copy over non-standard fields from the primary branch catalog on request. 2178 bfields = [] 2179 for fname in project.header_propagate_fields: 2180 bfields.extend(prim_branch_cat.header.select_fields(fname)) 2181 cfields = [] 2182 for fname in project.header_propagate_fields: 2183 cfields.extend(summit_cat.header.select_fields(fname)) 2184 # Replace old with new set if not equal. 2185 if bfields != cfields: 2186 for cfield in cfields: 2187 summit_cat.header.field.remove(cfield) 2188 for bfield in bfields: 2189 summit_cat.header.field.append(bfield) 2190 2191 2192 _asc_check_cache = {} 2193 2194 def summit_scatter_single (branch_id, branch_name, branch_subdir, 2195 branch_path, summit_paths, 2196 project, options, update_progress): 2197 2198 update_progress() 2199 2200 # See if the branch catalog is to be newly created from the template. 2201 new_from_template = False 2202 branch_path_mod = branch_path 2203 if branch_path in project.add_on_scatter: 2204 new_from_template = True 2205 # Initialize new catalog with messages directly from the template. 2206 # Later the catalog file name will be switched to branch path, 2207 # if the catalog satisfies criteria to be created on scatter. 2208 branch_path_mod = project.add_on_scatter[branch_path] 2209 2210 # Open the branch catalog and all summit catalogs. 2211 try: 2212 branch_cat = Catalog(branch_path_mod, wrapping=project.branches_wrapping) 2213 except PologyError as e: 2214 warning(_("@info", 2215 "Cannot open the branch catalog '%(file)s' " 2216 "to scatter to. The error was:\n" 2217 "%(msg)s", 2218 file=branch_path_mod, msg=str_to_unicode(str(e)))) 2219 return 2220 summit_cats = [] 2221 for summit_path in summit_paths: 2222 try: 2223 # NOTE: Must be opened monitored to have compatible types 2224 # when copying message parts to branch message. 2225 summit_cat = Catalog(summit_path) 2226 except PologyError as e: 2227 warning(_("@info", 2228 "Cannot open the summit catalog '%(file)s' " 2229 "to scatter from. The error was:\n" 2230 "%(msg)s", 2231 file=summit_path, msg=str_to_unicode(str(e)))) 2232 return 2233 summit_cats.append(summit_cat) 2234 2235 # Collect and link ascription catalogs to summit catalogs. 2236 # (Do not open them here, but only later when a check is not cached.) 2237 if project.ascription_filter: 2238 aconfs_acats = {} 2239 for summit_cat in summit_cats: 2240 aconf, acatpath = project.aconfs_acatpaths[summit_cat.name] 2241 aconfs_acats[summit_cat.name] = (aconf, None, acatpath) 2242 if acatpath not in _asc_check_cache: 2243 _asc_check_cache[acatpath] = {} 2244 2245 # Pair branch messages with summit messages. 2246 msgs_total = 0 2247 msgs_translated = 0 2248 msg_links = [] 2249 asc_stopped = 0 2250 for branch_msg in branch_cat: 2251 update_progress() 2252 2253 # Skip obsolete messages. 2254 if branch_msg.obsolete: 2255 continue 2256 msgs_total += 1 2257 2258 # If there is a hook on branch messages on gather, 2259 # it must be used here to prepare branch message for lookup 2260 # in summit catalog, as the hook may modify the key. 2261 branch_msg_lkp = branch_msg 2262 if project.hook_on_gather_msg_branch: 2263 branch_msg_lkp = MessageUnsafe(branch_msg) 2264 exec_hook_msg(branch_id, branch_name, branch_subdir, 2265 branch_msg_lkp, branch_cat, 2266 project.hook_on_gather_msg_branch) 2267 2268 # Construct branch message for lookup with extended key. 2269 branch_xkmsg_lkp = extkey_msg(branch_msg_lkp) 2270 2271 # Find first summit catalog which has this message translated. 2272 summit_msg = None 2273 for summit_cat in summit_cats: 2274 # Branch message with extended key must be looked up first. 2275 for bmsg_lkp in [branch_xkmsg_lkp, branch_msg_lkp]: 2276 if bmsg_lkp in summit_cat: 2277 summit_msg = summit_cat[bmsg_lkp] 2278 if summit_msg.obsolete: 2279 summit_msg = None 2280 else: 2281 break 2282 if summit_msg is not None: 2283 break 2284 2285 if summit_msg is None: 2286 report_on_msg(_("@info:progress", 2287 "Message not in the summit."), 2288 branch_msg, branch_cat) 2289 continue 2290 2291 if ( project.ascription_filter and not options.force 2292 and do_scatter(summit_msg, branch_msg) 2293 ): 2294 aconf, acat, acatpath = aconfs_acats[summit_cat.name] 2295 if summit_msg.key not in _asc_check_cache[acatpath]: 2296 if acat is None: 2297 acat = Catalog(acatpath, monitored=False, create=True) 2298 aconfs_acats[summit_cat.name] = (aconf, acat, acatpath) 2299 hfilter = project.ascription_history_filter 2300 ahist = collect_ascription_history(summit_msg, acat, aconf, 2301 nomrg=True, hfilter=hfilter) 2302 afilter = project.ascription_filter 2303 res = afilter(summit_msg, summit_cat, ahist, aconf) 2304 _asc_check_cache[acatpath][summit_msg.key] = res 2305 if not _asc_check_cache[acatpath][summit_msg.key]: 2306 asc_stopped += 1 2307 continue 2308 2309 if summit_msg.translated: 2310 msgs_translated += 1 2311 msg_links.append((branch_msg, summit_msg, summit_cat)) 2312 2313 if asc_stopped > 0: 2314 warning(n_("@info:progress", 2315 "%(file)s: %(num)d message stopped by ascription filter.", 2316 "%(file)s: %(num)d messages stopped by ascription filter.", 2317 file=branch_path, num=asc_stopped)) 2318 2319 # If completeness less than minimal acceptable, remove all translations. 2320 if msgs_total > 0: 2321 completeness_ratio = float(msgs_translated) / msgs_total 2322 else: 2323 completeness_ratio = 1.0 2324 if ( completeness_ratio < project.scatter_acc_completeness 2325 and not options.force 2326 ): 2327 for branch_msg in branch_cat: 2328 if branch_msg.obsolete: 2329 branch_cat.remove_on_sync(branch_msg) 2330 else: 2331 clear_msg(branch_msg) 2332 2333 # If complete enough, scatter from summit to branch messages. 2334 else: 2335 scattered_branch_msgs = set() 2336 for branch_msg, summit_msg, summit_cat in msg_links: 2337 update_progress() 2338 2339 if do_scatter(summit_msg, branch_msg): 2340 exec_hook_msg(branch_id, branch_name, branch_subdir, 2341 summit_msg, summit_cat, 2342 project.hook_on_scatter_msg) 2343 2344 # NOTE: Same plurality and equal msgid_plural fields 2345 # between summit and branch message are enforced, 2346 # so only assert this for robustness. 2347 if summit_msg.msgid_plural != branch_msg.msgid_plural: 2348 error(_("@info", 2349 "Cannot scatter messages with " 2350 "different plurality.")) 2351 2352 for i in range(len(summit_msg.msgstr)): 2353 piped_msgstr = exec_hook_msgstr( 2354 branch_id, branch_name, branch_subdir, 2355 summit_msg.msgstr[i], summit_msg, summit_cat, 2356 project.hook_on_scatter_msgstr) 2357 if i < len(branch_msg.msgstr): 2358 branch_msg.msgstr[i] = piped_msgstr 2359 else: 2360 branch_msg.msgstr.append(piped_msgstr) 2361 branch_msg.unfuzzy() 2362 branch_msg.manual_comment = summit_msg.manual_comment 2363 scattered_branch_msgs.add(branch_msg) 2364 2365 # Fuzzy all active messages which were not scattered, 2366 # in order to avoid stale translations in branches. 2367 for branch_msg in branch_cat: 2368 if branch_msg.active and branch_msg not in scattered_branch_msgs: 2369 branch_msg.fuzzy = True 2370 branch_msg.msgctxt_previous = branch_msg.msgctxt 2371 branch_msg.msgid_previous = branch_msg.msgid 2372 branch_msg.msgid_plural_previous = branch_msg.msgid_plural 2373 2374 # Update branch header based on primary summit catalog. 2375 # Copy over all header parts from summit to branch, 2376 # except for those copied from template on merging. 2377 hdr = branch_cat.header 2378 shdr = summit_cats[0].header 2379 # Fields to keep due to being copied over on merging. 2380 keep_fields = [ 2381 "Report-Msgid-Bugs-To", 2382 "POT-Creation-Date", 2383 ] 2384 # Fields to keep if no branch message was modified. 2385 if not branch_cat.modcount and branch_cat.header.initialized: 2386 keep_fields.extend([ 2387 "PO-Revision-Date", 2388 "Last-Translator", 2389 ]) 2390 # Fields to keep due to explicitly being told to. 2391 keep_fields.extend(project.header_skip_fields_on_scatter) 2392 # Update comments. 2393 hdr.title = shdr.title 2394 hdr.copyright = shdr.copyright 2395 hdr.license = shdr.license 2396 hdr.author = shdr.author 2397 hdr.comment = shdr.comment 2398 # Update fields only if normalized lists of fields do not match. 2399 if normhf(hdr.field, keep_fields) != normhf(shdr.field, keep_fields): 2400 # Collect branch fields to be preserved. 2401 preserved_fs = [] 2402 for fnam in keep_fields: 2403 selected_fs = branch_cat.header.select_fields(fnam) 2404 preserved_fs.append(selected_fs[0] if selected_fs else (fnam, None)) 2405 # Overwrite branch with summit header fields. 2406 hdr.field = shdr.field 2407 # Put back the preserved branch fields. 2408 for fnam, fval in preserved_fs: 2409 if fval is not None: 2410 hdr.set_field(fnam, fval) 2411 else: 2412 hdr.remove_field(fnam) 2413 2414 # Apply hooks to the branch catalog. 2415 exec_hook_cat(branch_id, branch_name, branch_subdir, branch_cat, 2416 project.hook_on_scatter_cat) 2417 2418 # If the branch catalog has been newly created, 2419 # see if it is translated enough to be really written out. 2420 skip_write = False 2421 if new_from_template and not options.force: 2422 ntrans = 0 2423 for msg in branch_cat: 2424 if msg.translated: 2425 ntrans += 1 2426 if len(branch_cat) > 0: 2427 skip_write = ( float(ntrans) / len(branch_cat) + 1e-6 2428 < project.scatter_min_completeness) 2429 else: 2430 skip_write = False 2431 2432 if new_from_template and not skip_write: 2433 # Create any needed subdirectories and set destination branch path. 2434 mkdirpath(os.path.dirname(branch_path)) 2435 branch_cat.filename = branch_path 2436 2437 # Commit changes to the branch catalog. 2438 if not skip_write and (branch_cat.sync() or options.force): 2439 2440 # Apply hooks to branch catalog file. 2441 exec_hook_file(branch_id, branch_name, branch_subdir, 2442 branch_cat.filename, project.hook_on_scatter_file) 2443 2444 # Add to version control. 2445 if ( project.branches_vcs 2446 and not project.bdict[branch_id].skip_version_control 2447 ): 2448 if not project.branches_vcs.add(branch_cat.filename): 2449 warning(_("@info", 2450 "Cannot add '%(file)s' to version control.", 2451 file=branch_cat.filename)) 2452 2453 paths_str = " ".join(summit_paths) 2454 if options.verbose: 2455 if new_from_template: 2456 actype = _("@item:intext action performed on a catalog", 2457 "scattered-added") 2458 report("<+ (%s) %s %s" 2459 % (actype, branch_cat.filename, paths_str)) 2460 else: 2461 actype = _("@item:intext action performed on a catalog", 2462 "scattered") 2463 report("< (%s) %s %s" 2464 % (actype, branch_cat.filename, paths_str)) 2465 elif not options.quiet: 2466 if new_from_template: 2467 report("<+ %s %s" % (branch_cat.filename, paths_str)) 2468 else: 2469 report("< %s %s" % (branch_cat.filename, paths_str)) 2470 2471 2472 def do_scatter (smsg, bmsg): 2473 2474 return smsg.translated 2475 2476 2477 def hook_applicable (branch_check, branch_id, name_check, name, subdir): 2478 2479 if branch_check is not None: 2480 if hasattr(branch_check, "__call__"): 2481 if not branch_check(branch_id): 2482 return False 2483 else: 2484 if not re.search(branch_check, branch_id): 2485 return False 2486 2487 if name_check is not None: 2488 if hasattr(name_check, "__call__"): 2489 if not name_check(name, subdir): 2490 return False 2491 else: 2492 if not re.search(name_check, name): 2493 return False 2494 2495 return True 2496 2497 2498 # Pipe msgstr through hook calls, 2499 # for which branch id and catalog name match hook specification. 2500 def exec_hook_msgstr (branch_id, branch_name, branch_subdir, 2501 msgstr, msg, cat, hooks): 2502 2503 piped_msgstr = msgstr 2504 for call, branch_ch, name_ch in hooks: 2505 if hook_applicable(branch_ch, branch_id, name_ch, 2506 branch_name, branch_subdir): 2507 piped_msgstr_tmp = call(piped_msgstr, msg, cat) 2508 if isinstance(piped_msgstr_tmp, str): 2509 piped_msgstr = piped_msgstr_tmp 2510 2511 return piped_msgstr 2512 2513 2514 # Pipe message through hook calls, 2515 # for which branch id and catalog name match hook specification. 2516 def exec_hook_msg (branch_id, branch_name, branch_subdir, msg, cat, hooks): 2517 2518 # Apply all hooks to the message. 2519 for call, branch_ch, name_ch in hooks: 2520 if hook_applicable(branch_ch, branch_id, name_ch, 2521 branch_name, branch_subdir): 2522 call(msg, cat) 2523 2524 2525 # Pipe header through hook calls, 2526 # for which branch id and catalog name match hook specification. 2527 def exec_hook_head (branch_id, branch_name, branch_subdir, hdr, cat, hooks): 2528 2529 # Apply all hooks to the header. 2530 for call, branch_ch, name_ch in hooks: 2531 if hook_applicable(branch_ch, branch_id, name_ch, 2532 branch_name, branch_subdir): 2533 call(hdr, cat) 2534 2535 2536 # Pipe catalog through hook calls, 2537 # for which branch id and catalog name match hook specification. 2538 def exec_hook_cat (branch_id, branch_name, branch_subdir, cat, hooks): 2539 2540 # Apply all hooks to the catalog. 2541 for call, branch_ch, name_ch in hooks: 2542 if hook_applicable(branch_ch, branch_id, name_ch, 2543 branch_name, branch_subdir): 2544 call(cat) 2545 2546 2547 # Pipe catalog file through hook calls, 2548 # for which branch id and catalog name match hook specification. 2549 def exec_hook_file (branch_id, branch_name, branch_subdir, filepath, hooks): 2550 2551 # Make temporary backup of the file. 2552 # FIXME: Portable construction of temporary file. 2553 bckppath = "/tmp/backup%s-%s" % (os.getpid(), os.path.basename(filepath)) 2554 shutil.copyfile(filepath, bckppath) 2555 2556 # Apply all hooks to the file, but stop if one returns non-zero status. 2557 failed = False 2558 for call, branch_ch, name_ch in hooks: 2559 if hook_applicable(branch_ch, branch_id, name_ch, 2560 branch_name, branch_subdir): 2561 if call(filepath) != 0: 2562 failed = True 2563 break 2564 2565 # If any hook failed, retrieve the temporary copy. 2566 if failed: 2567 shutil.move(bckppath, filepath) 2568 else: 2569 os.unlink(bckppath) 2570 2571 2572 # Pipe branch through hook calls, 2573 # for which branch id and matches hook specification. 2574 def exec_hook_branch (branch_id, hooks): 2575 2576 # Apply all hooks to the branch, but stop if one returns non-zero status. 2577 failed = False 2578 for call, branch_ch, d1 in hooks: 2579 if hook_applicable(branch_ch, branch_id, None, None, None): 2580 if call(branch_id) != 0: 2581 failed = True 2582 break 2583 2584 2585 def find_summit_comment (msg, summit_tag): 2586 2587 i = 0 2588 for c in msg.auto_comment: 2589 if c.startswith(summit_tag): 2590 return i 2591 i += 1 2592 return -1 2593 2594 2595 def get_summit_comment (msg, summit_tag, default=""): 2596 2597 p = find_summit_comment(msg, summit_tag) 2598 if p >= 0: 2599 return msg.auto_comment[p][len(summit_tag):].strip() 2600 else: 2601 return default 2602 2603 2604 def set_summit_comment (msg, summit_tag, text): 2605 2606 ctext = str(summit_tag + " " + text.strip()) 2607 p = find_summit_comment(msg, summit_tag) 2608 if p >= 0: 2609 msg.auto_comment[p] = ctext 2610 else: 2611 msg.auto_comment.append(ctext) 2612 2613 2614 _summit_tag_branchid = "+>" 2615 _summit_tag_kwprop = "+:" 2616 _summit_tags = ( 2617 _summit_tag_branchid, 2618 _summit_tag_kwprop, 2619 ) 2620 2621 def summit_set_tags (msg, branch_id, project): 2622 2623 # Add branch ID. 2624 branch_ids = get_summit_comment(msg, _summit_tag_branchid, "").split() 2625 if branch_id not in branch_ids: 2626 branch_ids.append(branch_id) 2627 set_summit_comment(msg, _summit_tag_branchid, " ".join(branch_ids)) 2628 2629 2630 def summit_override_auto (summit_msg, branch_msg, branch_id, is_primary): 2631 2632 # Copy auto/source/flag comments only if this is the primary branch 2633 # for the current message. 2634 if is_primary: 2635 2636 # Equalize flags, except the fuzzy. 2637 for fl in branch_msg.flag: 2638 if fl != "fuzzy": 2639 summit_msg.flag.add(fl) 2640 for fl in summit_msg.flag: 2641 if fl != "fuzzy" and fl not in branch_msg.flag: 2642 summit_msg.flag.remove(fl) 2643 2644 # Equalize source references. 2645 # FIXME: Once there is a way to reliably tell the root directory 2646 # of source references, add missing and remove obsolete source 2647 # references instead. 2648 summit_msg.source = Monlist(list(map(Monpair, branch_msg.source))) 2649 2650 # Split auto comments of the current summit message into 2651 # summit and non-summit tagged comments. 2652 # Also of the branch message, in case it has summit-alike comments. 2653 summit_nscmnts, summit_scmnts = split_summit_comments(summit_msg) 2654 branch_nscmnts, branch_scmnts = split_summit_comments(branch_msg) 2655 2656 # Override auto comments only if different overally 2657 # (which needs not be, due to double fresh/old insertion) 2658 # and non-summit auto comments of the current summit message 2659 # are different to the branch message auto comments. 2660 if ( summit_msg.auto_comment != branch_msg.auto_comment 2661 and summit_nscmnts != branch_nscmnts 2662 ): 2663 summit_msg.auto_comment = Monlist(branch_msg.auto_comment) 2664 summit_msg.auto_comment.extend(summit_scmnts) 2665 2666 2667 def split_summit_comments (msg): 2668 2669 non_summit_comments = [] 2670 summit_comments = [] 2671 for comment in msg.auto_comment: 2672 wlst = comment.split() 2673 if wlst and wlst[0] in _summit_tags: 2674 summit_comments.append(comment) 2675 else: 2676 non_summit_comments.append(comment) 2677 2678 return non_summit_comments, summit_comments 2679 2680 2681 def summit_merge_single (branch_id, catalog_name, catalog_subdir, 2682 catalog_path, template_path, 2683 wrapping, fuzzy_merging, 2684 project, options, update_progress): 2685 2686 update_progress() 2687 2688 # Gather the summit template in summit-over-dynamic-templates mode. 2689 if project.templates_dynamic and branch_id == SUMMIT_ID: 2690 summit_gather_single(catalog_name, project.tproject, project.toptions, 2691 update_progress=update_progress) 2692 2693 # FIXME: Portable construction of temporary file. 2694 tmp_path = os.path.join("/tmp", ( os.path.basename(catalog_path) 2695 + "~merged-%d" % os.getpid())) 2696 2697 # Whether to create pristine catalog from template. 2698 vivified = catalog_path in project.add_on_merge 2699 2700 # Skip calling msgmerge if template creation dates exist and are equal. 2701 do_msgmerge = True 2702 if not vivified and not project.templates_dynamic and not options.force: 2703 hdr = Catalog(catalog_path, monitored=False, headonly=True).header 2704 thdr = Catalog(template_path, monitored=False, headonly=True).header 2705 pcd = hdr.get_field_value("POT-Creation-Date") 2706 tpcd = thdr.get_field_value("POT-Creation-Date") 2707 do_msgmerge = (not pcd or not tpcd or pcd != tpcd) 2708 2709 header_prop_fields = project.header_propagate_fields 2710 2711 # Should merged catalog be opened, and in what mode? 2712 do_open = False 2713 headonly = False 2714 monitored = False 2715 otherwrap = set(wrapping).difference(["basic"]) 2716 if otherwrap or project.hook_on_merge_msg or project.hook_on_merge_cat: 2717 do_open = True 2718 elif header_prop_fields or project.hook_on_merge_head or vivified: 2719 do_open = True 2720 headonly = True 2721 if ( header_prop_fields or vivified 2722 or project.hook_on_merge_head or project.hook_on_merge_msg 2723 or project.hook_on_merge_cat 2724 ): 2725 monitored = True 2726 2727 # Should template catalog be opened too? 2728 do_open_template = False 2729 if header_prop_fields or vivified: 2730 do_open_template = True 2731 2732 cat = None 2733 if do_msgmerge: 2734 # Create the temporary merged catalog. 2735 minasfz, refuzzy = 0.0, False 2736 cmppaths, fuzzex, minwnex = [], False, 0 2737 if branch_id == SUMMIT_ID: 2738 minasfz = project.merge_min_adjsim_fuzzy 2739 refuzzy = project.merge_rebase_fuzzy 2740 if project.compendium_on_merge: 2741 cmppaths.append(project.compendium_on_merge) 2742 fuzzex = project.compendium_fuzzy_exact 2743 minwnex = project.compendium_min_words_exact 2744 catalog_path_mod = catalog_path 2745 if vivified: 2746 if cmppaths: 2747 catalog_path_mod = "/dev/null" 2748 else: 2749 catalog_path_mod = tmp_path 2750 shutil.copyfile(template_path, tmp_path) 2751 2752 getcat = do_open and not headonly 2753 ignpotdate = project.templates_dynamic 2754 cat = merge_pofile(catalog_path_mod, template_path, outpath=tmp_path, 2755 wrapping=wrapping, fuzzymatch=fuzzy_merging, 2756 minasfz=minasfz, refuzzy=refuzzy, 2757 cmppaths=cmppaths, fuzzex=fuzzex, minwnex=minwnex, 2758 getcat=getcat, monitored=monitored, 2759 ignpotdate=ignpotdate, 2760 quiet=True, abort=False) 2761 if cat is None: 2762 warning(_("@info", 2763 "Catalog '%(file1)s' not merged with " 2764 "template '%(file2)s' due to errors on merging.", 2765 file1=catalog_path_mod, file2=template_path)) 2766 return 2767 elif not getcat: 2768 # Catalog not requested, so the return value is True 2769 # indicating that the merge succedded. 2770 cat = None 2771 2772 else: 2773 # Copy current to temporary catalog, to be processed by hooks, etc. 2774 shutil.copyfile(catalog_path, tmp_path) 2775 2776 # Save good time by opening the merged catalog only if necessary, 2777 # and only as much as necessary. 2778 2779 # Open catalogs as necessary. 2780 if do_open: 2781 update_progress() 2782 if cat is None: 2783 cat = Catalog(tmp_path, monitored=monitored, wrapping=wrapping, 2784 headonly=headonly) 2785 if do_open_template: 2786 tcat = Catalog(template_path, monitored=False, headonly=True) 2787 2788 # Initialize header if the catalog has been vivified from template. 2789 if vivified: 2790 hdr = cat.header 2791 hdr.title = Monlist() 2792 hdr.copyright = "" 2793 hdr.license = "" 2794 hdr.author = Monlist() 2795 hdr.comment = Monlist() 2796 # Get the project ID from template; 2797 # if it gives default value, use catalog name instead. 2798 projid = tcat.header.get_field_value("Project-Id-Version") 2799 if not projid or "PACKAGE" in projid: 2800 projid = catalog_name 2801 hdr.set_field("Project-Id-Version", str(projid)) 2802 rdate = time.strftime("%Y-%m-%d %H:%M%z") 2803 hdr.set_field("PO-Revision-Date", str(rdate)) 2804 hdr.set_field("Last-Translator", str(project.vivify_w_translator)) 2805 hdr.set_field("Language-Team", str(project.vivify_w_langteam)) 2806 if project.vivify_w_language: 2807 hdr.set_field("Language", str(project.vivify_w_language), 2808 after="Language-Team", reorder=True) 2809 hdr.set_field("Content-Type", 2810 "text/plain; charset=%s" % project.vivify_w_charset) 2811 hdr.set_field("Content-Transfer-Encoding", "8bit") 2812 if project.vivify_w_plurals: 2813 hdr.set_field("Plural-Forms", str(project.vivify_w_plurals)) 2814 else: 2815 hdr.remove_field("Plural-Forms") 2816 2817 # Propagate requested header fields. 2818 if header_prop_fields: 2819 # Preserve order of the fields when collecting. 2820 fields = [] 2821 for field in cat.header.field: 2822 if field[0] in header_prop_fields: 2823 fields.append(field) 2824 tfields = [] 2825 for tfield in tcat.header.field: 2826 if tfield[0] in header_prop_fields: 2827 tfields.append(tfield) 2828 # Replace the field sequence if not equal to that of the template. 2829 if fields != tfields: 2830 for field in fields: 2831 cat.header.field.remove(field) 2832 for tfield in tfields: 2833 cat.header.field.append(tfield) 2834 2835 # Set original instead of temporary file path -- hooks may expect it. 2836 if cat is not None: 2837 cat.filename = catalog_path 2838 2839 # Execute header hooks. 2840 if project.hook_on_merge_head: 2841 exec_hook_head(branch_id, catalog_name, catalog_subdir, 2842 cat.header, cat, project.hook_on_merge_head) 2843 2844 # Execute message hooks. 2845 if project.hook_on_merge_msg: 2846 for msg in cat: 2847 exec_hook_msg(branch_id, catalog_name, catalog_subdir, 2848 msg, cat, project.hook_on_merge_msg) 2849 2850 # Execute catalog hooks. 2851 if project.hook_on_merge_cat: 2852 exec_hook_cat(branch_id, catalog_name, catalog_subdir, 2853 cat, project.hook_on_merge_cat) 2854 2855 # Synchronize merged catalog if it has been opened. 2856 if cat is not None: 2857 cat.filename = tmp_path # not to overwrite original file 2858 cat.sync(force=otherwrap) 2859 2860 # Execute file hooks. 2861 if project.hook_on_merge_file: 2862 cat_name = os.path.basename(tmp_path) 2863 cat_name = cat_name[:cat_name.rfind(".po")] 2864 exec_hook_file(branch_id, cat_name, catalog_subdir, tmp_path, 2865 project.hook_on_merge_file) 2866 2867 # If there is any difference between merged and old catalog. 2868 if vivified or not filecmp.cmp(catalog_path, tmp_path): 2869 # Assert correctness of the merged catalog and move over the old. 2870 assert_system("msgfmt -c -o/dev/null %s " % tmp_path) 2871 added = False 2872 if vivified: 2873 added = True 2874 mkdirpath(os.path.dirname(catalog_path)) 2875 shutil.move(tmp_path, catalog_path) 2876 2877 # Add to version control if not already added. 2878 vcs = project.summit_vcs if SUMMIT_ID else project.branches_vcs 2879 if ( vcs 2880 and ( branch_id == SUMMIT_ID or 2881 not project.bdict[branch_id].skip_version_control) 2882 and not vcs.is_versioned(catalog_path) 2883 ): 2884 if not vcs.add(catalog_path): 2885 warning(_("@info", 2886 "Cannot add '%(file)s' to version control.", 2887 file=catalog_path)) 2888 2889 if options.verbose: 2890 if added: 2891 actype = _("@item:intext action performed on a catalog", 2892 "merged-added") 2893 report(".+ (%s) %s" % (actype, catalog_path)) 2894 else: 2895 actype = _("@item:intext action performed on a catalog", 2896 "merged") 2897 report(". (%s) %s" % (actype, catalog_path)) 2898 elif not options.quiet: 2899 if added: 2900 report(".+ %s" % catalog_path) 2901 else: 2902 report(". %s" % catalog_path) 2903 2904 # Remove the temporary merged catalog. 2905 if os.path.exists(tmp_path): 2906 os.remove(tmp_path) 2907 2908 2909 # Put header fields in canonical form, for equality checking. 2910 # Returns ordered list of (field name, field value). 2911 def normhf (fields, excluded=[]): 2912 2913 nfs = [] 2914 2915 for fnam, fval in fields: 2916 if fnam not in excluded: 2917 nfs.append((fnam, fval)) 2918 nfs.sort() 2919 2920 return nfs 2921 2922 2923 # Remove all translator-related elements from the message. 2924 def clear_msg (msg): 2925 2926 msg.unfuzzy() 2927 msg.msgstr[:] = [""] * len(msg.msgstr) 2928 msg.manual_comment[:] = [] 2929 2930 return msg 2931 2932 2933 if __name__ == '__main__': 2934 exit_on_exception(main)