Merge, filter, and sort file entries from multiple files

bin/backup

@@ -17,8 +17,12 @@ files are hardlinked into the new snapshot.
 import argparse
 import logging
 import os
+import re
 import sys
+from contextlib import contextmanager
+from pathlib import Path
 from subprocess import CalledProcessError
+from tempfile import NamedTemporaryFile
 
 from rbackup.rsync import rsync
 from rbackup.struct.repository import Repository
@@ -46,10 +50,9 @@ EXTRA_RSYNC_OPTS = {
 
 # ----- File Options -----
 CONFIG_DIR = "/etc/rbackup"
-FILE_OPTS = [
-    f"--files-from={CONFIG_DIR}/include-paths.conf",
-    f"--exclude-from={CONFIG_DIR}/home-exclude.conf",
-]
+FILE_OPTS = [f"--exclude-from={CONFIG_DIR}/home-exclude.conf"]
+INCLUDE_PATHS = [f"{CONFIG_DIR}/etc-include.conf", "{CONFIG_DIR}/system-include.conf"]
+COMMENT_REGEX = r"^[^#; ]+"
 
 
 # ----- Error Codes -----
@@ -116,6 +119,43 @@ def parse_cmdline_arguments(**kwargs):
     return parser.parse_args(**kwargs)
 
 
+@contextmanager
+def merge_files(*config_files):
+    """Parse, filter, and sort through config files to create a single
+    --files-from argument.
+
+        Any files included that do not exist send a warning to the log.
+
+    >>> merge_files('/etc/rbackup/etc-include.conf', '/etc/rbackup/system-include.conf') # doctest: +ELLIPSIS
+    >>> '/tmp/...'
+
+    :param config_files: files including paths to read from
+    :type config_files: iterable of str
+    :returns: path to file that lists include paths
+    :rtype: str
+    """
+    file_paths = [Path(p) for p in config_files]
+    include_lines = []
+
+    for config_file in file_paths:
+        if config_file.exists():
+            with config_file.open(mode="r") as opened_file:
+                include_lines.extend(
+                    l for l in opened_file.readlines() if re.match(COMMENT_REGEX, l)
+                )
+        else:
+            syslog.warning(f"{config_file} does not exist, ignoring")
+
+    include_lines.sort()
+
+    with NamedTemporaryFile(mode="w", delete=False) as include_paths:
+        include_paths.writelines(include_lines)
+
+    tmpfile = Path(include_paths.name)
+    yield tmpfile
+    tmpfile.unlink()
+
+
 # ========== Main Script ==========
 if __name__ == "__main__":
     args = parse_cmdline_arguments()
@@ -139,8 +179,16 @@ if __name__ == "__main__":
     curr_snapshot = None
 
     try:
-        curr_snapshot = repo.create_snapshot(args.name)
-        rsync(*rsync_opts, *FILE_OPTS, *link_dests, "/", str(curr_snapshot.path))
+        with merge_files(INCLUDE_PATHS) as include_file:
+            curr_snapshot = repo.create_snapshot(args.name)
+            rsync(
+                *rsync_opts,
+                *FILE_OPTS,
+                f"--files-from={include_file}",
+                *link_dests,
+                "/",
+                str(curr_snapshot.path),
+            )
     except ValueError as e:
         syslog.critical(e)
         exit(E_INVALID_SNAPSHOT_NAME)