Warning, /sdk/pology/bin/posieve is written in an unsupported language. File is not indexed.

0001 #!/usr/bin/env python3
0002 # -*- coding: UTF-8 -*-
0003 
0004 """
0005 Sieve messages in collections of PO files.
0006 
0007 Reference documentation in C{doc/user/sieving.docbook}.
0008 
0009 @author: Chusslove Illich (Часлав Илић) <caslav.ilic@gmx.net>
0010 @license: GPLv3
0011 """
0012 
0013 try:
0014     import fallback_import_paths
0015 except:
0016     pass
0017 
0018 import glob
0019 import locale
0020 import os
0021 import re
0022 import sys
0023 from types import ModuleType
0024 
0025 from pology import datadir, version, _, n_, t_
0026 from pology.catalog import Catalog, CatalogSyntaxError
0027 from pology.colors import ColorOptionParser, set_coloring_globals
0028 import pology.config as pology_config
0029 from pology.escape import escape_sh
0030 from pology.fsops import str_to_unicode, unicode_to_str
0031 from pology.fsops import collect_catalogs, collect_system
0032 from pology.fsops import build_path_selector, collect_paths_from_file
0033 from pology.fsops import collect_paths_cmdline
0034 from pology.fsops import exit_on_exception
0035 from pology.msgreport import report_on_msg, warning_on_msg, error_on_msg
0036 from pology.report import error, warning, report, encwrite
0037 from pology.report import init_file_progress
0038 from pology.report import list_options
0039 from pology.report import format_item_list
0040 from pology.stdcmdopt import add_cmdopt_filesfrom, add_cmdopt_incexc
0041 from pology.stdcmdopt import add_cmdopt_colors
0042 from pology.subcmd import ParamParser
0043 from pology.sieve import SieveMessageError, SieveCatalogError
0044 
0045 
0046 def main ():
0047 
0048     locale.setlocale(locale.LC_ALL, "")
0049 
0050     # Get defaults for command line options from global config.
0051     cfgsec = pology_config.section("posieve")
0052     def_do_skip = cfgsec.boolean("skip-on-error", True)
0053     def_msgfmt_check = cfgsec.boolean("msgfmt-check", False)
0054     def_skip_obsolete = cfgsec.boolean("skip-obsolete", False)
0055 
0056     # Setup options and parse the command line.
0057     usage = _("@info command usage",
0058         "%(cmd)s [OPTIONS] SIEVE [POPATHS...]",
0059         cmd="%prog")
0060     desc = _("@info command description",
0061         "Apply sieves to PO paths, which may be either single PO files or "
0062         "directories to search recursively for PO files. "
0063         "Some of the sieves only examine PO files, while others "
0064         "modify them as well. "
0065         "The first non-option argument is the sieve name; "
0066         "a list of several comma-separated sieves can be given too.")
0067     ver = _("@info command version",
0068         "%(cmd)s (Pology) %(version)s\n"
0069         "Copyright © 2007, 2008, 2009, 2010 "
0070         "Chusslove Illich (Часлав Илић) &lt;%(email)s&gt;",
0071         cmd="%prog", version=version(), email="caslav.ilic@gmx.net")
0072 
0073     opars = ColorOptionParser(usage=usage, description=desc, version=ver)
0074     opars.add_option(
0075         "-a", "--announce-entry",
0076         action="store_true", dest="announce_entry", default=False,
0077         help=_("@info command line option description",
0078                "Announce that header or message is just about to be sieved."))
0079     opars.add_option(
0080         "-b", "--skip-obsolete",
0081         action="store_true", dest="skip_obsolete", default=def_skip_obsolete,
0082         help=_("@info command line option description",
0083                "Do not sieve obsolete messages."))
0084     opars.add_option(
0085         "-c", "--msgfmt-check",
0086         action="store_true", dest="msgfmt_check", default=def_msgfmt_check,
0087         help=_("@info command line option description",
0088                "Check catalogs by %(cmd)s and skip those which do not pass.",
0089                cmd="msgfmt -c"))
0090     opars.add_option(
0091         "-u", "--single-entry",
0092         metavar=_("@info command line value placeholder", "ENTRY_NUMBER"),
0093         action="store", dest="single_entry", default=0,
0094         help=_("@info command line option description",
0095                "Only perform the check on this ENTRY_NUMBER."))
0096     opars.add_option(
0097         "--force-sync",
0098         action="store_true", dest="force_sync", default=False,
0099         help=_("@info command line option description",
0100                "Force rewriting of all messages, whether modified or not."))
0101     opars.add_option(
0102         "-H", "--help-sieves",
0103         action="store_true", dest="help_sieves", default=False,
0104         help=_("@info command line option description",
0105                "Show help for applied sieves."))
0106     opars.add_option(
0107         "--issued-params",
0108         action="store_true", dest="issued_params", default=False,
0109         help=_("@info command line option description",
0110                "Show all issued sieve parameters "
0111                "(from command line and user configuration)."))
0112     opars.add_option(
0113         "-l", "--list-sieves",
0114         action="store_true", dest="list_sieves", default=False,
0115         help=_("@info command line option description",
0116                "List available internal sieves."))
0117     opars.add_option(
0118         "--list-options",
0119         action="store_true", dest="list_options", default=False,
0120         help=_("@info command line option description",
0121                "List the names of available options."))
0122     opars.add_option(
0123         "--list-sieve-names",
0124         action="store_true", dest="list_sieve_names", default=False,
0125         help=_("@info command line option description",
0126                "List the names of available internal sieves."))
0127     opars.add_option(
0128         "--list-sieve-params",
0129         action="store_true", dest="list_sieve_params", default=False,
0130         help=_("@info command line option description",
0131                "List the parameters known to issued sieves."))
0132     opars.add_option(
0133         "-m", "--output-modified",
0134         metavar=_("@info command line value placeholder", "FILE"),
0135         action="store", dest="output_modified", default=None,
0136         help=_("@info command line option description",
0137                "Output names of modified files into FILE."))
0138     opars.add_option(
0139         "--no-skip",
0140         action="store_false", dest="do_skip", default=def_do_skip,
0141         help=_("@info command line option description",
0142                "Do not try to skip catalogs which signal errors."))
0143     opars.add_option(
0144         "--no-sync",
0145         action="store_false", dest="do_sync", default=True,
0146         help=_("@info command line option description",
0147                "Do not write any modifications to catalogs."))
0148     opars.add_option(
0149         "-q", "--quiet",
0150         action="store_true", dest="quiet", default=False,
0151         help=_("@info command line option description",
0152                "Do not display any progress info "
0153                "(does not influence sieves themselves)."))
0154     opars.add_option(
0155         "-s",
0156         metavar=_("@info command line value placeholder", "NAME[:VALUE]"),
0157         action="append", dest="sieve_params", default=[],
0158         help=_("@info command line option description",
0159                "Pass a parameter to sieves."))
0160     opars.add_option(
0161         "-S",
0162         metavar=_("@info command line value placeholder", "NAME[:VALUE]"),
0163         action="append", dest="sieve_no_params", default=[],
0164         help=_("@info command line option description",
0165                "Remove a parameter to sieves "
0166                "(e.g. if it was issued through user configuration)."))
0167     opars.add_option(
0168         "-v", "--verbose",
0169         action="store_true", dest="verbose", default=False,
0170         help=_("@info command line option description",
0171                "Output more detailed progress information."))
0172     add_cmdopt_filesfrom(opars)
0173     add_cmdopt_incexc(opars)
0174     add_cmdopt_colors(opars)
0175 
0176     (op, free_args) = opars.parse_args(str_to_unicode(sys.argv[1:]))
0177 
0178     if op.list_options:
0179         report(list_options(opars))
0180         sys.exit(0)
0181 
0182     if len(free_args) < 1 and not (op.list_sieves or op.list_sieve_names):
0183         error(_("@info", "No sieve to apply given."))
0184 
0185     op.raw_sieves = []
0186     op.raw_paths = []
0187     if len(free_args) > 2 and op.single_entry != 0:
0188         error(_("@info", "With single entry mode, you can only give one input file."))
0189 
0190     if len(free_args) >= 1:
0191         op.raw_sieves = free_args[0]
0192         op.raw_paths = free_args[1:]
0193 
0194     # Could use some speedup.
0195     try:
0196         import psyco
0197         psyco.full()
0198     except ImportError:
0199         pass
0200 
0201     set_coloring_globals(ctype=op.coloring_type, outdep=(not op.raw_colors))
0202 
0203     # Dummy-set all internal sieves as requested if sieve listing required.
0204     sieves_requested = []
0205     if op.list_sieves or op.list_sieve_names:
0206         # Global sieves.
0207         modpaths = glob.glob(os.path.join(datadir(), "sieve", "[a-z]*.py"))
0208         modpaths.sort()
0209         for modpath in modpaths:
0210             sname = os.path.basename(modpath)[:-3] # minus .py
0211             sname = sname.replace("_", "-")
0212             sieves_requested.append(sname)
0213         # Language-specific sieves.
0214         modpaths = glob.glob(os.path.join(datadir(),
0215                                           "lang", "*", "sieve", "[a-z]*.py"))
0216         modpaths.sort()
0217         for modpath in modpaths:
0218             sname = os.path.basename(modpath)[:-3] # minus .py
0219             sname = sname.replace("_", "-")
0220             lang = os.path.basename(os.path.dirname(os.path.dirname(modpath)))
0221             sieves_requested.append(lang + ":" + sname)
0222 
0223     # No need to load and setup sieves if only listing sieve names requested.
0224     if op.list_sieve_names:
0225         report("\n".join(sieves_requested))
0226         sys.exit(0)
0227 
0228     # Load sieve modules from supplied names in the command line.
0229     if not sieves_requested:
0230         sieves_requested = op.raw_sieves.split(",")
0231     sieve_modules = []
0232     for sieve_name in sieves_requested:
0233         # Resolve sieve file.
0234         if not sieve_name.endswith(".py"):
0235             # One of internal sieves.
0236             if ":" in sieve_name:
0237                 # Language-specific internal sieve.
0238                 lang, name = sieve_name.split(":")
0239                 sieve_path_base = os.path.join("lang", lang, "sieve", name)
0240             else:
0241                 sieve_path_base = os.path.join("sieve", sieve_name)
0242             sieve_path_base = sieve_path_base.replace("-", "_") + ".py"
0243             sieve_path = os.path.join(datadir(), sieve_path_base)
0244         else:
0245             # Sieve name is its path.
0246             sieve_path = sieve_name
0247         try:
0248             with open(unicode_to_str(sieve_path)) as sieve_file:
0249                 sieve_code = sieve_file.read()
0250             # ...unicode_to_str because of exec below.
0251         except IOError:
0252             error(_("@info",
0253                     "Cannot load sieve '%(file)s'.",
0254                     file=sieve_path))
0255         # Load file into new module.
0256         sieve_mod_name = "sieve_" + str(len(sieve_modules))
0257         sieve_mod = ModuleType(sieve_mod_name)
0258         exec(sieve_code, sieve_mod.__dict__)
0259         sys.modules[sieve_mod_name] = sieve_mod # to avoid garbage collection
0260         sieve_modules.append((sieve_name, sieve_mod))
0261         if not hasattr(sieve_mod, "Sieve"):
0262             error(_("@info",
0263                     "Module '%(file)s' does not define %(classname)s class.",
0264                     file=sieve_path, classname="Sieve"))
0265 
0266     # Setup sieves (description, known parameters...)
0267     pp = ParamParser()
0268     snames = []
0269     for name, mod in sieve_modules:
0270         scview = pp.add_subcmd(name)
0271         if hasattr(mod, "setup_sieve"):
0272             mod.setup_sieve(scview)
0273         snames.append(name)
0274 
0275     # If info on sieves requested, report and exit.
0276     if op.list_sieves:
0277         report(_("@info", "Available internal sieves:"))
0278         report(pp.listcmd(snames))
0279         sys.exit(0)
0280     elif op.list_sieve_params:
0281         params = set()
0282         for scview in pp.cmdviews():
0283             params.update(scview.params(addcol=True))
0284         report("\n".join(sorted(params)))
0285         sys.exit(0)
0286     elif op.help_sieves:
0287         report(_("@info", "Help for sieves:"))
0288         report("")
0289         report(pp.help(snames))
0290         sys.exit(0)
0291 
0292     # Prepare sieve parameters for parsing.
0293     sieve_params = list(op.sieve_params)
0294     # - append paramaters according to configuration
0295     sieve_params.extend(read_config_params(pp.cmdviews(), sieve_params))
0296     # - remove paramaters according to command line
0297     if op.sieve_no_params:
0298         sieve_params_mod = []
0299         for parspec in sieve_params:
0300             if parspec.split(":", 1)[0] not in op.sieve_no_params:
0301                 sieve_params_mod.append(parspec)
0302         sieve_params = sieve_params_mod
0303 
0304     # If assembly of issued parameters requested, report and exit.
0305     if op.issued_params:
0306         escparams = []
0307         for parspec in sieve_params:
0308             if ":" in parspec:
0309                 param, value = parspec.split(":", 1)
0310                 escparam = "%s:%s" % (param, escape_sh(value))
0311             else:
0312                 escparam = parspec
0313             escparams.append(escparam)
0314         fmtparams = " ".join(["-s%s" % x for x in sorted(escparams)])
0315         if fmtparams:
0316             report(fmtparams)
0317         sys.exit(0)
0318 
0319     # Parse sieve parameters.
0320     sparams, nacc_params = pp.parse(sieve_params, snames)
0321     if nacc_params:
0322         error(_("@info",
0323                 "Parameters not accepted by any of issued subcommands: "
0324                 "%(paramlist)s.",
0325                 paramlist=format_item_list(nacc_params)))
0326 
0327     # ========================================
0328     # FIXME: Think of something less ugly.
0329     # Add as special parameter to each sieve:
0330     # - root paths from which the catalogs are collected
0331     # - whether destination independent coloring is in effect
0332     # - test function for catalog selection
0333     root_paths = []
0334     if op.raw_paths:
0335         root_paths.extend(op.raw_paths)
0336     if op.files_from:
0337         for ffpath in op.files_from:
0338             root_paths.extend(collect_paths_from_file(ffpath))
0339     if not op.raw_paths and not op.files_from:
0340         root_paths = ["."]
0341     is_cat_included = build_path_selector(incnames=op.include_names,
0342                                           incpaths=op.include_paths,
0343                                           excnames=op.exclude_names,
0344                                           excpaths=op.exclude_paths)
0345     for p in list(sparams.values()):
0346         p.root_paths = root_paths
0347         p.raw_colors = op.raw_colors
0348         p.is_cat_included = is_cat_included
0349     # ========================================
0350 
0351     # Create sieves.
0352     sieves = []
0353     for name, mod in sieve_modules:
0354         sieves.append(mod.Sieve(sparams[name]))
0355 
0356     # Get the message monitoring indicator from the sieves.
0357     # Monitor unless all sieves have requested otherwise.
0358     use_monitored = False
0359     for sieve in sieves:
0360         if getattr(sieve, "caller_monitored", True):
0361             use_monitored = True
0362             break
0363     if op.verbose and not use_monitored:
0364         report(_("@info:progress", "--> Not monitoring messages."))
0365 
0366     # Get the sync indicator from the sieves.
0367     # Sync unless all sieves have requested otherwise,
0368     # and unless syncing is disabled globally in command line.
0369     do_sync = False
0370     for sieve in sieves:
0371         if getattr(sieve, "caller_sync", True):
0372             do_sync = True
0373             break
0374     if not op.do_sync:
0375         do_sync = False
0376     if op.verbose and not do_sync:
0377         report(_("@info:progress", "--> Not syncing after sieving."))
0378 
0379     # Open in header-only mode if no sieve has message processor.
0380     # Categorize sieves by the presence of message/header processors.
0381     use_headonly = True
0382     header_sieves = []
0383     header_sieves_last = []
0384     message_sieves = []
0385     for sieve in sieves:
0386         if hasattr(sieve, "process"):
0387             use_headonly = False
0388             message_sieves.append(sieve)
0389         if hasattr(sieve, "process_header"):
0390             header_sieves.append(sieve)
0391         if hasattr(sieve, "process_header_last"):
0392             header_sieves_last.append(sieve)
0393     if op.verbose and use_headonly:
0394         report(_("@info:progress", "--> Opening catalogs in header-only mode."))
0395 
0396     # Collect catalog paths.
0397     fnames = collect_paths_cmdline(rawpaths=op.raw_paths,
0398                                    incnames=op.include_names,
0399                                    incpaths=op.include_paths,
0400                                    excnames=op.exclude_names,
0401                                    excpaths=op.exclude_paths,
0402                                    filesfrom=op.files_from,
0403                                    elsecwd=True,
0404                                    respathf=collect_catalogs,
0405                                    abort=True)
0406 
0407     if op.do_skip:
0408         errwarn = warning
0409         errwarn_on_msg = warning_on_msg
0410     else:
0411         errwarn = error
0412         errwarn_on_msg = error_on_msg
0413 
0414     # Prepare inline progress indicator.
0415     if not op.quiet:
0416         update_progress = init_file_progress(fnames,
0417             addfmt=t_("@info:progress", "Sieving: %(file)s"))
0418 
0419     # Sieve catalogs.
0420     modified_files = []
0421     for fname in fnames:
0422         if op.verbose:
0423             report(_("@info:progress", "Sieving %(file)s...", file=fname))
0424         elif not op.quiet:
0425             update_progress(fname)
0426 
0427         if op.msgfmt_check:
0428             d1, oerr, ret = collect_system(["msgfmt", "-o", "/dev/null", "-c",
0429                                             fname])
0430             if ret != 0:
0431                 oerr = oerr.strip()
0432                 errwarn(_("@info:progress",
0433                           "%(file)s: %(cmd)s check failed:\n"
0434                           "%(msg)s",
0435                           file=fname, cmd="msgfmt -c", msg=oerr))
0436                 warning(_("@info:progress",
0437                           "Skipping catalog due to syntax check failure."))
0438                 continue
0439 
0440         try:
0441             cat = Catalog(fname, monitored=use_monitored, headonly=use_headonly, single_entry=int(op.single_entry))
0442         except CatalogSyntaxError as e:
0443             errwarn(_("@info:progress",
0444                       "%(file)s: Parsing failed: %(msg)s",
0445                       file=fname, msg=e))
0446             warning(_("@info:progress",
0447                       "Skipping catalog due to parsing failure."))
0448             continue
0449 
0450         skip = False
0451         # First run all header sieves.
0452         if header_sieves and op.announce_entry:
0453             report(_("@info:progress",
0454                      "Sieving header of %(file)s...", file=fname))
0455         for sieve in header_sieves:
0456             try:
0457                 ret = sieve.process_header(cat.header, cat)
0458             except SieveCatalogError as e:
0459                 errwarn(_("@info:progress",
0460                           "%(file)s:header: Sieving failed: %(msg)s",
0461                           file=fname, msg=e))
0462                 skip = True
0463                 break
0464             if ret not in (None, 0):
0465                 break
0466         if skip:
0467             warning(_("@info:progress",
0468                       "Skipping catalog due to header sieving failure."))
0469             continue
0470 
0471         # Then run all message sieves on each message,
0472         # unless processing only the header.
0473         if not use_headonly:
0474             for msg in cat:
0475                 if op.skip_obsolete and msg.obsolete:
0476                     continue
0477 
0478                 if not op.quiet:
0479                     update_progress(fname)
0480 
0481                 if op.announce_entry:
0482                     report(_("@info:progress",
0483                              "Sieving %(file)s:%(line)d(#%(entry)d)...",
0484                              file=fname, line=msg.refline, entry=msg.refentry))
0485 
0486                 for sieve in message_sieves:
0487                     try:
0488                         ret = sieve.process(msg, cat)
0489                     except SieveMessageError as e:
0490                         errwarn_on_msg(_("@info:progress",
0491                                          "Sieving failed: %(msg)s", msg=e),
0492                                          msg, cat)
0493                         break
0494                     except SieveCatalogError as e:
0495                         errwarn_on_msg(_("@info:progress",
0496                                          "Sieving failed: %(msg)s", msg=e),
0497                                          msg, cat)
0498                         skip = True
0499                         break
0500                     if ret not in (None, 0):
0501                         break
0502                 if skip:
0503                     break
0504         if skip:
0505             warning(_("@info:progress",
0506                       "Skipping catalog due to message sieving failure."))
0507             continue
0508 
0509         # Finally run all header-last sieves.
0510         if header_sieves_last and op.announce_entry:
0511             report(_("@info:progress",
0512                      "Sieving header (after messages) in %(file)s...",
0513                      file=fname))
0514         for sieve in header_sieves_last:
0515             try:
0516                 ret = sieve.process_header_last(cat.header, cat)
0517             except SieveCatalogError as e:
0518                 errwarn(_("@info:progress",
0519                           "%(file)s:header: Sieving (after messages) "
0520                           "failed: %(msg)s",
0521                           file=fname, msg=e))
0522                 skip = True
0523                 break
0524             if ret not in (None, 0):
0525                 break
0526         if skip:
0527             warning(_("@info:progress",
0528                       "Skipping catalog due to header sieving "
0529                       "(after messages) failure."))
0530             continue
0531 
0532         if do_sync and cat.sync(op.force_sync):
0533             if op.verbose:
0534                 report(_("@info:progress leading ! is a shorthand "
0535                          "state indicator",
0536                          "! (MODIFIED) %(file)s",
0537                          file=fname))
0538             elif not op.quiet:
0539                 report(_("@info:progress leading ! is a shorthand "
0540                          "state indicator",
0541                          "! %(file)s",
0542                          file=fname))
0543             modified_files.append(fname)
0544 
0545     if not op.quiet:
0546         update_progress() # clear last progress line, if any
0547 
0548     for sieve in sieves:
0549         if hasattr(sieve, "finalize"):
0550             try:
0551                 sieve.finalize()
0552             except SieveCatalogError as e:
0553                 warning(_("@info:progress",
0554                           "Finalization failed: %(msg)s",
0555                           msg=e))
0556 
0557     if op.output_modified:
0558         ofh = open(op.output_modified, "w")
0559         ofh.write("\n".join(modified_files) + "\n")
0560         ofh.close
0561 
0562 
0563 def read_config_params (scviews, cmdline_parspecs):
0564 
0565     # Collect parameters defined in the config.
0566     cfgsec = pology_config.section("posieve")
0567     pref = "param-"
0568     config_params = []
0569     for field in cfgsec.fields():
0570         if field.startswith(pref):
0571             parspec = field[len(pref):]
0572             only_sieves = None
0573             inverted = False
0574             if "/" in parspec:
0575                 param, svspec = parspec.split("/", 1)
0576                 if svspec.startswith("~"):
0577                     inverted = True
0578                     svspec = svspec[1:]
0579                 only_sieves = set(svspec.split(","))
0580             else:
0581                 param = parspec
0582             if "." in param:
0583                 param, d1 = param.split(".", 1)
0584             config_params.append((field, param, only_sieves, inverted))
0585 
0586     if not config_params:
0587         return []
0588 
0589     # Collect parameters known to issued sieves and issued in command line.
0590     sieves = set([x.name() for x in scviews])
0591     acc_raw_params = set(sum([x.params(addcol=True) for x in scviews], []))
0592     acc_params = set([x.rstrip(":") for x in acc_raw_params])
0593     acc_flag_params = set([x for x in acc_raw_params if not x.endswith(":")])
0594     cmd_params = set([x.split(":", 1)[0] for x in cmdline_parspecs])
0595 
0596     # Select parameters based on issued sieves.
0597     sel_params = []
0598     for field, param, only_sieves, inverted in config_params:
0599         if param in acc_params and param not in cmd_params:
0600             if only_sieves is not None:
0601                 overlap = bool(sieves.intersection(only_sieves))
0602                 add_param = overlap if not inverted else not overlap
0603             else:
0604                 add_param = True
0605             if add_param:
0606                 if param in acc_flag_params:
0607                     if cfgsec.boolean(field):
0608                         sel_params.append(param)
0609                 else:
0610                     sel_params.append("%s:%s" % (param, cfgsec.string(field)))
0611 
0612     return sel_params
0613 
0614 
0615 if __name__ == '__main__':
0616     exit_on_exception(main)