Warning, /sdk/pology/bin/posieve is written in an unsupported language. File is not indexed.
0001 #!/usr/bin/env python3
0002 # -*- coding: UTF-8 -*-
0003
0004 """
0005 Sieve messages in collections of PO files.
0006
0007 Reference documentation in C{doc/user/sieving.docbook}.
0008
0009 @author: Chusslove Illich (Часлав Илић) <caslav.ilic@gmx.net>
0010 @license: GPLv3
0011 """
0012
0013 try:
0014 import fallback_import_paths
0015 except:
0016 pass
0017
0018 import glob
0019 import locale
0020 import os
0021 import re
0022 import sys
0023 from types import ModuleType
0024
0025 from pology import datadir, version, _, n_, t_
0026 from pology.catalog import Catalog, CatalogSyntaxError
0027 from pology.colors import ColorOptionParser, set_coloring_globals
0028 import pology.config as pology_config
0029 from pology.escape import escape_sh
0030 from pology.fsops import str_to_unicode, unicode_to_str
0031 from pology.fsops import collect_catalogs, collect_system
0032 from pology.fsops import build_path_selector, collect_paths_from_file
0033 from pology.fsops import collect_paths_cmdline
0034 from pology.fsops import exit_on_exception
0035 from pology.msgreport import report_on_msg, warning_on_msg, error_on_msg
0036 from pology.report import error, warning, report, encwrite
0037 from pology.report import init_file_progress
0038 from pology.report import list_options
0039 from pology.report import format_item_list
0040 from pology.stdcmdopt import add_cmdopt_filesfrom, add_cmdopt_incexc
0041 from pology.stdcmdopt import add_cmdopt_colors
0042 from pology.subcmd import ParamParser
0043 from pology.sieve import SieveMessageError, SieveCatalogError
0044
0045
0046 def main ():
0047
0048 locale.setlocale(locale.LC_ALL, "")
0049
0050 # Get defaults for command line options from global config.
0051 cfgsec = pology_config.section("posieve")
0052 def_do_skip = cfgsec.boolean("skip-on-error", True)
0053 def_msgfmt_check = cfgsec.boolean("msgfmt-check", False)
0054 def_skip_obsolete = cfgsec.boolean("skip-obsolete", False)
0055
0056 # Setup options and parse the command line.
0057 usage = _("@info command usage",
0058 "%(cmd)s [OPTIONS] SIEVE [POPATHS...]",
0059 cmd="%prog")
0060 desc = _("@info command description",
0061 "Apply sieves to PO paths, which may be either single PO files or "
0062 "directories to search recursively for PO files. "
0063 "Some of the sieves only examine PO files, while others "
0064 "modify them as well. "
0065 "The first non-option argument is the sieve name; "
0066 "a list of several comma-separated sieves can be given too.")
0067 ver = _("@info command version",
0068 "%(cmd)s (Pology) %(version)s\n"
0069 "Copyright © 2007, 2008, 2009, 2010 "
0070 "Chusslove Illich (Часлав Илић) <%(email)s>",
0071 cmd="%prog", version=version(), email="caslav.ilic@gmx.net")
0072
0073 opars = ColorOptionParser(usage=usage, description=desc, version=ver)
0074 opars.add_option(
0075 "-a", "--announce-entry",
0076 action="store_true", dest="announce_entry", default=False,
0077 help=_("@info command line option description",
0078 "Announce that header or message is just about to be sieved."))
0079 opars.add_option(
0080 "-b", "--skip-obsolete",
0081 action="store_true", dest="skip_obsolete", default=def_skip_obsolete,
0082 help=_("@info command line option description",
0083 "Do not sieve obsolete messages."))
0084 opars.add_option(
0085 "-c", "--msgfmt-check",
0086 action="store_true", dest="msgfmt_check", default=def_msgfmt_check,
0087 help=_("@info command line option description",
0088 "Check catalogs by %(cmd)s and skip those which do not pass.",
0089 cmd="msgfmt -c"))
0090 opars.add_option(
0091 "-u", "--single-entry",
0092 metavar=_("@info command line value placeholder", "ENTRY_NUMBER"),
0093 action="store", dest="single_entry", default=0,
0094 help=_("@info command line option description",
0095 "Only perform the check on this ENTRY_NUMBER."))
0096 opars.add_option(
0097 "--force-sync",
0098 action="store_true", dest="force_sync", default=False,
0099 help=_("@info command line option description",
0100 "Force rewriting of all messages, whether modified or not."))
0101 opars.add_option(
0102 "-H", "--help-sieves",
0103 action="store_true", dest="help_sieves", default=False,
0104 help=_("@info command line option description",
0105 "Show help for applied sieves."))
0106 opars.add_option(
0107 "--issued-params",
0108 action="store_true", dest="issued_params", default=False,
0109 help=_("@info command line option description",
0110 "Show all issued sieve parameters "
0111 "(from command line and user configuration)."))
0112 opars.add_option(
0113 "-l", "--list-sieves",
0114 action="store_true", dest="list_sieves", default=False,
0115 help=_("@info command line option description",
0116 "List available internal sieves."))
0117 opars.add_option(
0118 "--list-options",
0119 action="store_true", dest="list_options", default=False,
0120 help=_("@info command line option description",
0121 "List the names of available options."))
0122 opars.add_option(
0123 "--list-sieve-names",
0124 action="store_true", dest="list_sieve_names", default=False,
0125 help=_("@info command line option description",
0126 "List the names of available internal sieves."))
0127 opars.add_option(
0128 "--list-sieve-params",
0129 action="store_true", dest="list_sieve_params", default=False,
0130 help=_("@info command line option description",
0131 "List the parameters known to issued sieves."))
0132 opars.add_option(
0133 "-m", "--output-modified",
0134 metavar=_("@info command line value placeholder", "FILE"),
0135 action="store", dest="output_modified", default=None,
0136 help=_("@info command line option description",
0137 "Output names of modified files into FILE."))
0138 opars.add_option(
0139 "--no-skip",
0140 action="store_false", dest="do_skip", default=def_do_skip,
0141 help=_("@info command line option description",
0142 "Do not try to skip catalogs which signal errors."))
0143 opars.add_option(
0144 "--no-sync",
0145 action="store_false", dest="do_sync", default=True,
0146 help=_("@info command line option description",
0147 "Do not write any modifications to catalogs."))
0148 opars.add_option(
0149 "-q", "--quiet",
0150 action="store_true", dest="quiet", default=False,
0151 help=_("@info command line option description",
0152 "Do not display any progress info "
0153 "(does not influence sieves themselves)."))
0154 opars.add_option(
0155 "-s",
0156 metavar=_("@info command line value placeholder", "NAME[:VALUE]"),
0157 action="append", dest="sieve_params", default=[],
0158 help=_("@info command line option description",
0159 "Pass a parameter to sieves."))
0160 opars.add_option(
0161 "-S",
0162 metavar=_("@info command line value placeholder", "NAME[:VALUE]"),
0163 action="append", dest="sieve_no_params", default=[],
0164 help=_("@info command line option description",
0165 "Remove a parameter to sieves "
0166 "(e.g. if it was issued through user configuration)."))
0167 opars.add_option(
0168 "-v", "--verbose",
0169 action="store_true", dest="verbose", default=False,
0170 help=_("@info command line option description",
0171 "Output more detailed progress information."))
0172 add_cmdopt_filesfrom(opars)
0173 add_cmdopt_incexc(opars)
0174 add_cmdopt_colors(opars)
0175
0176 (op, free_args) = opars.parse_args(str_to_unicode(sys.argv[1:]))
0177
0178 if op.list_options:
0179 report(list_options(opars))
0180 sys.exit(0)
0181
0182 if len(free_args) < 1 and not (op.list_sieves or op.list_sieve_names):
0183 error(_("@info", "No sieve to apply given."))
0184
0185 op.raw_sieves = []
0186 op.raw_paths = []
0187 if len(free_args) > 2 and op.single_entry != 0:
0188 error(_("@info", "With single entry mode, you can only give one input file."))
0189
0190 if len(free_args) >= 1:
0191 op.raw_sieves = free_args[0]
0192 op.raw_paths = free_args[1:]
0193
0194 # Could use some speedup.
0195 try:
0196 import psyco
0197 psyco.full()
0198 except ImportError:
0199 pass
0200
0201 set_coloring_globals(ctype=op.coloring_type, outdep=(not op.raw_colors))
0202
0203 # Dummy-set all internal sieves as requested if sieve listing required.
0204 sieves_requested = []
0205 if op.list_sieves or op.list_sieve_names:
0206 # Global sieves.
0207 modpaths = glob.glob(os.path.join(datadir(), "sieve", "[a-z]*.py"))
0208 modpaths.sort()
0209 for modpath in modpaths:
0210 sname = os.path.basename(modpath)[:-3] # minus .py
0211 sname = sname.replace("_", "-")
0212 sieves_requested.append(sname)
0213 # Language-specific sieves.
0214 modpaths = glob.glob(os.path.join(datadir(),
0215 "lang", "*", "sieve", "[a-z]*.py"))
0216 modpaths.sort()
0217 for modpath in modpaths:
0218 sname = os.path.basename(modpath)[:-3] # minus .py
0219 sname = sname.replace("_", "-")
0220 lang = os.path.basename(os.path.dirname(os.path.dirname(modpath)))
0221 sieves_requested.append(lang + ":" + sname)
0222
0223 # No need to load and setup sieves if only listing sieve names requested.
0224 if op.list_sieve_names:
0225 report("\n".join(sieves_requested))
0226 sys.exit(0)
0227
0228 # Load sieve modules from supplied names in the command line.
0229 if not sieves_requested:
0230 sieves_requested = op.raw_sieves.split(",")
0231 sieve_modules = []
0232 for sieve_name in sieves_requested:
0233 # Resolve sieve file.
0234 if not sieve_name.endswith(".py"):
0235 # One of internal sieves.
0236 if ":" in sieve_name:
0237 # Language-specific internal sieve.
0238 lang, name = sieve_name.split(":")
0239 sieve_path_base = os.path.join("lang", lang, "sieve", name)
0240 else:
0241 sieve_path_base = os.path.join("sieve", sieve_name)
0242 sieve_path_base = sieve_path_base.replace("-", "_") + ".py"
0243 sieve_path = os.path.join(datadir(), sieve_path_base)
0244 else:
0245 # Sieve name is its path.
0246 sieve_path = sieve_name
0247 try:
0248 with open(unicode_to_str(sieve_path)) as sieve_file:
0249 sieve_code = sieve_file.read()
0250 # ...unicode_to_str because of exec below.
0251 except IOError:
0252 error(_("@info",
0253 "Cannot load sieve '%(file)s'.",
0254 file=sieve_path))
0255 # Load file into new module.
0256 sieve_mod_name = "sieve_" + str(len(sieve_modules))
0257 sieve_mod = ModuleType(sieve_mod_name)
0258 exec(sieve_code, sieve_mod.__dict__)
0259 sys.modules[sieve_mod_name] = sieve_mod # to avoid garbage collection
0260 sieve_modules.append((sieve_name, sieve_mod))
0261 if not hasattr(sieve_mod, "Sieve"):
0262 error(_("@info",
0263 "Module '%(file)s' does not define %(classname)s class.",
0264 file=sieve_path, classname="Sieve"))
0265
0266 # Setup sieves (description, known parameters...)
0267 pp = ParamParser()
0268 snames = []
0269 for name, mod in sieve_modules:
0270 scview = pp.add_subcmd(name)
0271 if hasattr(mod, "setup_sieve"):
0272 mod.setup_sieve(scview)
0273 snames.append(name)
0274
0275 # If info on sieves requested, report and exit.
0276 if op.list_sieves:
0277 report(_("@info", "Available internal sieves:"))
0278 report(pp.listcmd(snames))
0279 sys.exit(0)
0280 elif op.list_sieve_params:
0281 params = set()
0282 for scview in pp.cmdviews():
0283 params.update(scview.params(addcol=True))
0284 report("\n".join(sorted(params)))
0285 sys.exit(0)
0286 elif op.help_sieves:
0287 report(_("@info", "Help for sieves:"))
0288 report("")
0289 report(pp.help(snames))
0290 sys.exit(0)
0291
0292 # Prepare sieve parameters for parsing.
0293 sieve_params = list(op.sieve_params)
0294 # - append paramaters according to configuration
0295 sieve_params.extend(read_config_params(pp.cmdviews(), sieve_params))
0296 # - remove paramaters according to command line
0297 if op.sieve_no_params:
0298 sieve_params_mod = []
0299 for parspec in sieve_params:
0300 if parspec.split(":", 1)[0] not in op.sieve_no_params:
0301 sieve_params_mod.append(parspec)
0302 sieve_params = sieve_params_mod
0303
0304 # If assembly of issued parameters requested, report and exit.
0305 if op.issued_params:
0306 escparams = []
0307 for parspec in sieve_params:
0308 if ":" in parspec:
0309 param, value = parspec.split(":", 1)
0310 escparam = "%s:%s" % (param, escape_sh(value))
0311 else:
0312 escparam = parspec
0313 escparams.append(escparam)
0314 fmtparams = " ".join(["-s%s" % x for x in sorted(escparams)])
0315 if fmtparams:
0316 report(fmtparams)
0317 sys.exit(0)
0318
0319 # Parse sieve parameters.
0320 sparams, nacc_params = pp.parse(sieve_params, snames)
0321 if nacc_params:
0322 error(_("@info",
0323 "Parameters not accepted by any of issued subcommands: "
0324 "%(paramlist)s.",
0325 paramlist=format_item_list(nacc_params)))
0326
0327 # ========================================
0328 # FIXME: Think of something less ugly.
0329 # Add as special parameter to each sieve:
0330 # - root paths from which the catalogs are collected
0331 # - whether destination independent coloring is in effect
0332 # - test function for catalog selection
0333 root_paths = []
0334 if op.raw_paths:
0335 root_paths.extend(op.raw_paths)
0336 if op.files_from:
0337 for ffpath in op.files_from:
0338 root_paths.extend(collect_paths_from_file(ffpath))
0339 if not op.raw_paths and not op.files_from:
0340 root_paths = ["."]
0341 is_cat_included = build_path_selector(incnames=op.include_names,
0342 incpaths=op.include_paths,
0343 excnames=op.exclude_names,
0344 excpaths=op.exclude_paths)
0345 for p in list(sparams.values()):
0346 p.root_paths = root_paths
0347 p.raw_colors = op.raw_colors
0348 p.is_cat_included = is_cat_included
0349 # ========================================
0350
0351 # Create sieves.
0352 sieves = []
0353 for name, mod in sieve_modules:
0354 sieves.append(mod.Sieve(sparams[name]))
0355
0356 # Get the message monitoring indicator from the sieves.
0357 # Monitor unless all sieves have requested otherwise.
0358 use_monitored = False
0359 for sieve in sieves:
0360 if getattr(sieve, "caller_monitored", True):
0361 use_monitored = True
0362 break
0363 if op.verbose and not use_monitored:
0364 report(_("@info:progress", "--> Not monitoring messages."))
0365
0366 # Get the sync indicator from the sieves.
0367 # Sync unless all sieves have requested otherwise,
0368 # and unless syncing is disabled globally in command line.
0369 do_sync = False
0370 for sieve in sieves:
0371 if getattr(sieve, "caller_sync", True):
0372 do_sync = True
0373 break
0374 if not op.do_sync:
0375 do_sync = False
0376 if op.verbose and not do_sync:
0377 report(_("@info:progress", "--> Not syncing after sieving."))
0378
0379 # Open in header-only mode if no sieve has message processor.
0380 # Categorize sieves by the presence of message/header processors.
0381 use_headonly = True
0382 header_sieves = []
0383 header_sieves_last = []
0384 message_sieves = []
0385 for sieve in sieves:
0386 if hasattr(sieve, "process"):
0387 use_headonly = False
0388 message_sieves.append(sieve)
0389 if hasattr(sieve, "process_header"):
0390 header_sieves.append(sieve)
0391 if hasattr(sieve, "process_header_last"):
0392 header_sieves_last.append(sieve)
0393 if op.verbose and use_headonly:
0394 report(_("@info:progress", "--> Opening catalogs in header-only mode."))
0395
0396 # Collect catalog paths.
0397 fnames = collect_paths_cmdline(rawpaths=op.raw_paths,
0398 incnames=op.include_names,
0399 incpaths=op.include_paths,
0400 excnames=op.exclude_names,
0401 excpaths=op.exclude_paths,
0402 filesfrom=op.files_from,
0403 elsecwd=True,
0404 respathf=collect_catalogs,
0405 abort=True)
0406
0407 if op.do_skip:
0408 errwarn = warning
0409 errwarn_on_msg = warning_on_msg
0410 else:
0411 errwarn = error
0412 errwarn_on_msg = error_on_msg
0413
0414 # Prepare inline progress indicator.
0415 if not op.quiet:
0416 update_progress = init_file_progress(fnames,
0417 addfmt=t_("@info:progress", "Sieving: %(file)s"))
0418
0419 # Sieve catalogs.
0420 modified_files = []
0421 for fname in fnames:
0422 if op.verbose:
0423 report(_("@info:progress", "Sieving %(file)s...", file=fname))
0424 elif not op.quiet:
0425 update_progress(fname)
0426
0427 if op.msgfmt_check:
0428 d1, oerr, ret = collect_system(["msgfmt", "-o", "/dev/null", "-c",
0429 fname])
0430 if ret != 0:
0431 oerr = oerr.strip()
0432 errwarn(_("@info:progress",
0433 "%(file)s: %(cmd)s check failed:\n"
0434 "%(msg)s",
0435 file=fname, cmd="msgfmt -c", msg=oerr))
0436 warning(_("@info:progress",
0437 "Skipping catalog due to syntax check failure."))
0438 continue
0439
0440 try:
0441 cat = Catalog(fname, monitored=use_monitored, headonly=use_headonly, single_entry=int(op.single_entry))
0442 except CatalogSyntaxError as e:
0443 errwarn(_("@info:progress",
0444 "%(file)s: Parsing failed: %(msg)s",
0445 file=fname, msg=e))
0446 warning(_("@info:progress",
0447 "Skipping catalog due to parsing failure."))
0448 continue
0449
0450 skip = False
0451 # First run all header sieves.
0452 if header_sieves and op.announce_entry:
0453 report(_("@info:progress",
0454 "Sieving header of %(file)s...", file=fname))
0455 for sieve in header_sieves:
0456 try:
0457 ret = sieve.process_header(cat.header, cat)
0458 except SieveCatalogError as e:
0459 errwarn(_("@info:progress",
0460 "%(file)s:header: Sieving failed: %(msg)s",
0461 file=fname, msg=e))
0462 skip = True
0463 break
0464 if ret not in (None, 0):
0465 break
0466 if skip:
0467 warning(_("@info:progress",
0468 "Skipping catalog due to header sieving failure."))
0469 continue
0470
0471 # Then run all message sieves on each message,
0472 # unless processing only the header.
0473 if not use_headonly:
0474 for msg in cat:
0475 if op.skip_obsolete and msg.obsolete:
0476 continue
0477
0478 if not op.quiet:
0479 update_progress(fname)
0480
0481 if op.announce_entry:
0482 report(_("@info:progress",
0483 "Sieving %(file)s:%(line)d(#%(entry)d)...",
0484 file=fname, line=msg.refline, entry=msg.refentry))
0485
0486 for sieve in message_sieves:
0487 try:
0488 ret = sieve.process(msg, cat)
0489 except SieveMessageError as e:
0490 errwarn_on_msg(_("@info:progress",
0491 "Sieving failed: %(msg)s", msg=e),
0492 msg, cat)
0493 break
0494 except SieveCatalogError as e:
0495 errwarn_on_msg(_("@info:progress",
0496 "Sieving failed: %(msg)s", msg=e),
0497 msg, cat)
0498 skip = True
0499 break
0500 if ret not in (None, 0):
0501 break
0502 if skip:
0503 break
0504 if skip:
0505 warning(_("@info:progress",
0506 "Skipping catalog due to message sieving failure."))
0507 continue
0508
0509 # Finally run all header-last sieves.
0510 if header_sieves_last and op.announce_entry:
0511 report(_("@info:progress",
0512 "Sieving header (after messages) in %(file)s...",
0513 file=fname))
0514 for sieve in header_sieves_last:
0515 try:
0516 ret = sieve.process_header_last(cat.header, cat)
0517 except SieveCatalogError as e:
0518 errwarn(_("@info:progress",
0519 "%(file)s:header: Sieving (after messages) "
0520 "failed: %(msg)s",
0521 file=fname, msg=e))
0522 skip = True
0523 break
0524 if ret not in (None, 0):
0525 break
0526 if skip:
0527 warning(_("@info:progress",
0528 "Skipping catalog due to header sieving "
0529 "(after messages) failure."))
0530 continue
0531
0532 if do_sync and cat.sync(op.force_sync):
0533 if op.verbose:
0534 report(_("@info:progress leading ! is a shorthand "
0535 "state indicator",
0536 "! (MODIFIED) %(file)s",
0537 file=fname))
0538 elif not op.quiet:
0539 report(_("@info:progress leading ! is a shorthand "
0540 "state indicator",
0541 "! %(file)s",
0542 file=fname))
0543 modified_files.append(fname)
0544
0545 if not op.quiet:
0546 update_progress() # clear last progress line, if any
0547
0548 for sieve in sieves:
0549 if hasattr(sieve, "finalize"):
0550 try:
0551 sieve.finalize()
0552 except SieveCatalogError as e:
0553 warning(_("@info:progress",
0554 "Finalization failed: %(msg)s",
0555 msg=e))
0556
0557 if op.output_modified:
0558 ofh = open(op.output_modified, "w")
0559 ofh.write("\n".join(modified_files) + "\n")
0560 ofh.close
0561
0562
0563 def read_config_params (scviews, cmdline_parspecs):
0564
0565 # Collect parameters defined in the config.
0566 cfgsec = pology_config.section("posieve")
0567 pref = "param-"
0568 config_params = []
0569 for field in cfgsec.fields():
0570 if field.startswith(pref):
0571 parspec = field[len(pref):]
0572 only_sieves = None
0573 inverted = False
0574 if "/" in parspec:
0575 param, svspec = parspec.split("/", 1)
0576 if svspec.startswith("~"):
0577 inverted = True
0578 svspec = svspec[1:]
0579 only_sieves = set(svspec.split(","))
0580 else:
0581 param = parspec
0582 if "." in param:
0583 param, d1 = param.split(".", 1)
0584 config_params.append((field, param, only_sieves, inverted))
0585
0586 if not config_params:
0587 return []
0588
0589 # Collect parameters known to issued sieves and issued in command line.
0590 sieves = set([x.name() for x in scviews])
0591 acc_raw_params = set(sum([x.params(addcol=True) for x in scviews], []))
0592 acc_params = set([x.rstrip(":") for x in acc_raw_params])
0593 acc_flag_params = set([x for x in acc_raw_params if not x.endswith(":")])
0594 cmd_params = set([x.split(":", 1)[0] for x in cmdline_parspecs])
0595
0596 # Select parameters based on issued sieves.
0597 sel_params = []
0598 for field, param, only_sieves, inverted in config_params:
0599 if param in acc_params and param not in cmd_params:
0600 if only_sieves is not None:
0601 overlap = bool(sieves.intersection(only_sieves))
0602 add_param = overlap if not inverted else not overlap
0603 else:
0604 add_param = True
0605 if add_param:
0606 if param in acc_flag_params:
0607 if cfgsec.boolean(field):
0608 sel_params.append(param)
0609 else:
0610 sel_params.append("%s:%s" % (param, cfgsec.string(field)))
0611
0612 return sel_params
0613
0614
0615 if __name__ == '__main__':
0616 exit_on_exception(main)