PR for #139: Improving logging to avoid concurrency issues.

.github/checks/check_log_failures.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+import csv
+import sys
+from pathlib import Path
+
+RUN_CSV = Path("output") / "run.csv"
+
+def Main():
+    with open(RUN_CSV, newline="") as f:
+        rows = list(csv.DictReader(f))
+
+    script_failed = [row for row in rows if row["success"] != "1"]
+
+    sconstruct = Path("sconstruct.log").read_text()
+    scons_failed = (
+        "done building targets." not in sconstruct
+        or "errors occurred during build" in sconstruct
+        or "building terminated because of errors" in sconstruct
+    )
+
+    if not script_failed and not scons_failed:
+        print("All builds succeeded.")
+        return 0
+
+    print("Failed scripts:")
+    for row in script_failed:
+        print(" -", row["filename"])
+    if scons_failed:
+        print("SCons build failed. Check sconstruct.log for details.")
+    return 1
+
+if __name__ == "__main__":
+    sys.exit(Main())

.github/checks/checks.json

@@ -2,5 +2,5 @@
   {"name": "SCons DAG", "command": "/run-actions-dag"},
   {"name": "Newlines",  "command": "/run-actions-newlines"},
   {"name": "EPS data",  "command": "/run-actions-eps"},
-  {"name": "Build log", "command": "/run-actions-log"}
+  {"name": "Log failures", "command": "/run-actions-log"}
 ]

.github/workflows/checks.yml

@@ -78,13 +78,13 @@ jobs:
           display: EPS data
 
       - id: check_scons_log
-        name: Check SCons build log for errors
-        if: contains(fromJSON(steps.parse_commands.outputs.run), 'Build log')
+        name: Check log files for failures
+        if: contains(fromJSON(steps.parse_commands.outputs.run), 'Log failures')
         continue-on-error: true
         uses: ./.github/actions/timed-check
         with:
-          script: .github/checks/check_sconscript_log.py
-          display: Build log
+          script: .github/checks/check_log_failures.py
+          display: Log failures
 
       - name: Post results
         if: always()

.gitignore

@@ -1,6 +1,7 @@
 /drive
 /datastore
 /temp
+/log
 *.pyc
 *.lyx~
 *.lyx#

README.md

@@ -33,6 +33,8 @@ In addition, each project may use other specialized tools. For the working examp
 
 - `temp/` is not under version control; create the folder after cloning the repository.  `temp/` may used by scripts to store temporary or intermediate files (though some projects may not need it).
 
+- `log/` is not under version control; SCons builders write full per-script build logs there, mirroring `source/`. The root `sconstruct.log` remains the top-level build log.
+
 - _Issue folders_: When working on issue branches, you may create an issue folder at the top of the directory under version control (e.g. `./issue1_short_task_name`).
 
     - Code and (small) deliverables related to the issue are organized inside the issue folder. See [this example](https://github.com/JMSLab/Template/blob/05337cfa4a50ecfeda56afbdd295378d8e071a39/issue10_readme).
@@ -87,6 +89,8 @@ env.Stata(target, source)
 
 - `target` is a list with all of the files produced by the script.
 
+    - Build logs are not listed as SCons targets. Builder output is written automatically to `log/.../<script>.log`.
+
 - `source` is a list with the script's name and all of the files used as input; the script _must_ be the first element of the list.
 
 - `env.Stata` is the Stata builder provided in `source/lib/JMSLab/builders`; this is imported and saved as part of the `env` object in the `SConstruct` file at the root of the project.

SConstruct

@@ -26,6 +26,8 @@ env.Decider('MD5-timestamp') # Only computes hash if time-stamp changed
 Export('env', 'mode')
 
 jms.start_log('develop', '')
+jms.clean_orphaned_logs()
+atexit.register(jms.write_run_csv_from_log_dir)
 
 SConscript('source/derived/SConscript')
 SConscript('source/analysis/SConscript')