Merge pull request #270 from chetanya-goyal/main

Store simulation result file, return sim state from run

.github/scripts/expected_sim_outputs/temp-sense-gen/prePEX_sim_result

@@ -0,0 +1,8 @@
+Temp Frequency Power Error
+-20.0 129.13698399282515 0.0001395741 0.0
+0.0 812.2374949945864 0.000216328 -1.2889568113520866
+20.0 3726.282213709738 0.000308484 -1.905159990787503
+40.0 13548.077588756503 0.0004135831 -2.0993284842505844
+60.0 40753.5328218765 0.0005294722 -1.884757884614622
+80.0 104972.69292852504 0.0006544346 -1.288956811352108
+100.0 238491.19025467758 0.0007871842 -0.3737931886616934

.github/scripts/parse_rpt.py

@@ -28,8 +28,12 @@
 import json
 import os
 import re, subprocess
+import warnings
+
 sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+from common.get_ngspice_version import check_ngspice_version
 from common.check_gen_files import check_gen_files
+from common.check_sim_results import compare_files
 
 sys.stdout.flush()
 
@@ -45,6 +49,8 @@
 elif len(sys.argv) > 1:
     if sys.argv[1] == 'sky130hvl_ldo':
         _generator_is['sky130hvl_ldo'] = 1
+    elif sys.argv[1] == 'sky130hd_temp_full':
+        _generator_is['sky130hd_temp'] = 1
     else:
         _generator_is['sky130XX_cryo'] = 1
 
@@ -102,11 +108,52 @@
 
 ## Result File Check
 if _generator_is['sky130hvl_ldo']:
-   json_filename = "spec.json"
+    json_filename = "spec.json"
 else:
-   json_filename = "test.json"
+    json_filename = "test.json"
 
 if check_gen_files(json_filename, _generator_is, cryo_library):
-        print("Flow check is clean!")
+    print("Flow check is clean!")
 else:
     print("Flow check failed!")
+
+if len(sys.argv) > 1 and sys.argv[1] == "sky130hd_temp_full":
+    sim_state_filename = "work/sim_state_file.txt"
+    result_filename = "work/prePEX_sim_result" 
+    template_filename = "../../../.github/scripts/expected_sim_outputs/temp-sense-gen/prePEX_sim_result"
+    max_allowable_error = 0.5
+
+    ### Generated result file check against stored template
+    ngspice_version_flag = check_ngspice_version()
+    file_comp_flag = compare_files(template_filename, result_filename, max_allowable_error)
+
+    if ngspice_version_flag == 1 and file_comp_flag == 0:
+        raise ValueError("Ngspice version matches but sim results do not...sims failed!")
+    elif ngspice_version_flag == 0 and file_comp_flag == 0:
+        warnings.warn("The ngspice version does not match, "
+                       "simulation results might not match! "
+                       "Please contact a maintainer of the repo!", DeprecationWarning)
+    elif ngspice_version_flag == 0 and file_comp_flag == 1:
+        warnings.warn("The ngspice version does not match!", DeprecationWarning)
+
+    ### Number of failed simulations returned from simulation state check
+    sim_state = json.load(open("work/sim_state_file.txt"))
+    if sim_state["failed_sims"] != 0:
+        raise ValueError("Simulations failed: Non zero failed simulations!")
+
+    ### Generated file check
+    for folder_num in range(1, sim_state["completed_sims"] + 1):
+        dir_path = r'simulations/run/'
+        pex_path = os.listdir(dir_path)
+
+        file_name = "simulations/run/" + str(pex_path[0]) + "/" + str(folder_num) + "/"
+        param_file = file_name + "parameters.txt"
+        log_file = file_name + "sim_" + str(folder_num) + ".log"
+        spice_file = file_name + "sim_" + str(folder_num) + ".sp"
+
+        if os.path.exists(log_file) and os.path.exists(log_file) and os.path.exists(spice_file):
+            pass
+        else:
+            raise ValueError("Simulations failed: required number of run folders do not exist!")
+
+    print("Simulations are clean!")

.github/workflows/tempSense_sky130hd.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Checkout repo
         uses: actions/checkout@v2
 
-      - name: Test sky130hd Temp sensor
+      - name: Test sky130hd temperature sensor
         env:
                 IMAGE_NAME: msaligane/openfasoc:stable
         run: |
@@ -44,3 +44,24 @@ jobs:
             " && exit_code=$? | tee -a file.log
             if [ $? -ne 0 ]; then exit 1; fi
             if grep "\[ERROR\]" file.log; then exit 1; else exit 0; fi
+      
+      - name: Test sky130hd temperature sensor simulations
+        env:
+                IMAGE_NAME: msaligane/openfasoc:stable
+        run: |
+          cd $GITHUB_WORKSPACE
+          touch sim_file.log
+          docker run --rm \
+            -v $PWD:$PWD\
+            -w $PWD\
+            $IMAGE_NAME\
+            bash -c "\
+              cp ./.github/scripts/parse_rpt.py ./openfasoc/generators/temp-sense-gen/. &&\
+              pip3 install -r requirements.txt &&\
+              cd ./openfasoc/generators/temp-sense-gen &&\
+              make clean &&\
+              make sky130hd_temp_full &&\
+              python3 parse_rpt.py sky130hd_temp_full
+            " && exit_code=$? | tee -a file.log
+            if [ $? -ne 0 ]; then exit 1; fi
+            if grep "\[ERROR\]" file.log; then exit 1; else exit 0; fi

dependencies.sh

@@ -212,7 +212,7 @@ source ~/.bashrc
 if cat /etc/os-release | grep "ubuntu" >> /dev/null
 then
  apt install bison flex libx11-dev libx11-6 libxaw7-dev libreadline6-dev autoconf libtool automake -y
- git clone http://git.code.sf.net/p/ngspice/ngspice
+ git clone https://git.code.sf.net/p/ngspice/ngspice
  currentver="$(lsb_release -rs)"
  requiredver="22.04"
  if [ $currentver == $requiredver ]

openfasoc/generators/common/__init__.py

@@ -11,5 +11,8 @@
 	1. `check_gen_files(parameters: json_filename, is_tempsense)`: Used to check if the various files that should be generated by the flow are present in their required directories. 
 - `common.check_gen_extensions`
 	1. Stores the extensions of the files generated by the flow. 
+- `common.check_sim_results`
+	1. `compare_files(parameters: template_filename, result_filename, max_allowable error) -> int` Checks if the generated frequency, power and error result file values are within the maximum 
+ 	allowable error from those in the template file. Returns 1 for successful check, else returns 0
 See individual function documentation for more information on a particular function.
 """

openfasoc/generators/common/check_sim_results.py

@@ -0,0 +1,39 @@
+def compare_files(template_file, result_file, max_allowable_error) -> int:
+    """Checks if the generated simulation result file matches with 
+    the stored template file.
+
+    Args:
+    - 'template_file' (string): The stored template file
+    - 'result_file' (string): The result file generated by the simulations
+    - 'max_allowable_error' (float): The maximum allowable difference (percent error) in 
+    the values in both files
+
+    Returns:
+    - 'int': Returns 1 if the differences in readings (if any) is less than the 'max_allowable_error
+    else returns 0
+    """
+    with open(template_file, 'r') as template, open(result_file, 'r') as result:
+        next(template)
+        next(result)
+
+        for template_line, result_line in zip(template, result):
+            template_data = [float(val) for val in template_line.split()]
+            result_data = [float(val) for val in result_line.split()]
+
+            if template_data[1] != 0.0:
+                freq_diff = (abs(template_data[1] - result_data[1]) / template_data[1]) * 100
+            else:
+                freq_diff = (abs(template_data[1] - result_data[1])) * 100
+            if template_data[2] != 0.0:
+                power_diff = (abs(template_data[2] - result_data[2]) / template_data[2]) * 100
+            else:
+                power_diff = (abs(template_data[2] - result_data[2])) * 100
+            if template_data[3] != 0.0:
+                error_diff = (abs(template_data[3] - result_data[3]) / template_data[3]) * 100
+            else:
+                error_diff = (abs(template_data[3] - result_data[3])) * 100
+
+            if freq_diff <= max_allowable_error and power_diff <= max_allowable_error and error_diff <= max_allowable_error:
+                return 1
+            else:
+                return 0

openfasoc/generators/common/get_ngspice_version.py

@@ -0,0 +1,20 @@
+import subprocess
+import re
+
+def check_ngspice_version() -> int:
+    last_known_version = "41+"
+    result = subprocess.run(["ngspice", "--version"], capture_output=True, text=True)
+
+    if result.returncode == 0:
+        data = result.stdout.strip()
+        match = re.search(r'ngspice-(\S+)', data)
+
+        if match:
+            ngspice_version = match.group(1)
+            return ngspice_version == last_known_version
+        else:
+            print("Error parsing ngspice version.")
+    else:
+        print("Error getting ngspice version:", result.stderr)
+
+    return 0

openfasoc/generators/common/simulation/__init__.py

@@ -19,7 +19,7 @@ def run_simulations(
 	sim_tool: str = "ngspice",
 	num_concurrent_sims: int = 4,
 	netlist_path: str = "netlist.sp"
-) -> int:
+) -> dict:
 	"""Runs SPICE simulations.
 
     Generates configurations of all combinations of the given `parameters` and simulates each case. The testbench SPICE file, configuration parameters, and the output for each run are generated in the `simulation_dir/runs_dir` directory.
@@ -73,7 +73,7 @@ def run_simulations(
     - `num_concurrent_sims` (int = 4): The maximum number of concurrent simulations.
     - `netlist_path` (str = "netlist.sp"): Path to the SPICE netlist of the design to be simulated.
 
-    Returns (int): The number of simulations run.
+    Returns : A dictionary containing the number of ongoing (ideally 0), completed and failed simulations.
 	"""
 
 	runs_dir_path = path.join(simulation_dir, runs_dir)
@@ -94,11 +94,11 @@ def run_simulations(
 	print(f"Number of configurations: {config_number}")
 	print(f"Number of concurrent simulations: {num_concurrent_sims}")
 
-	_run_simulations(
+	sim_state = _run_simulations(
 		num_configs=config_number,
 		num_concurrent_sims=num_concurrent_sims,
 		sim_tool=sim_tool,
 		runs_dir_path=runs_dir_path
 	)
 
-	return config_number
+	return sim_state

openfasoc/generators/common/simulation/simulation_run.py

@@ -33,6 +33,8 @@ def _run_simulations(
     - `num_concurrent_sims` (int): The maximum number of concurrent simulations.
     - `sim_tool` (str): Path to the directory in which the simulation runs will be generated.
     - `runs_dir_path` (str): Path to the directory in which the simulation runs will be generated.
+
+    Returns: `simulation_state`, the number of ongoing, completed and failed sims.
 	"""
 
 	simulation_state = {
@@ -68,6 +70,7 @@ def thread_on_exit(exit_status: int, state=simulation_state):
 		time.sleep(1)
 
 	_print_progress(num_configs, simulation_state['completed_sims'], simulation_state['failed_sims'], start_time, end='\n')
+	return simulation_state
 
 def _run_config(
 	sim_tool: str,
@@ -148,4 +151,4 @@ def _threaded_run(
 	except:
 		return on_exit(1)
 
-	on_exit(0)
+	on_exit(0)

openfasoc/generators/temp-sense-gen/simulations/templates/tempsenseInst_ngspice.sp

@@ -46,7 +46,11 @@ vVSS                      VSS                      0                  dc
 
 c0 lc_out 0 1f
 
+.if (temp_var <= 20)
+.TRAN 1u 'sim_end'
+.else
 .TRAN 10n 'sim_end'
+.endif
 
 .meas   tran period TRIG v(lc_out) td=10p val=1.0 rise=2
 +           TARG v(lc_out) td=10p val=1.0 rise=3

openfasoc/generators/temp-sense-gen/tools/simulation.py

@@ -1,4 +1,4 @@
-import os
+import os, json
 import re
 import shutil
 import sys
@@ -57,7 +57,7 @@ def generate_runs(
 
         update_netlist(srcNetlist, dstNetlist, jsonConfig["simMode"])
 
-        num_simulations = run_simulations(
+        sim_state = run_simulations(
             parameters={
                 'temp': {'start': tempStart, 'end': tempStop, 'step': tempStep},
                 'model_file': model_file,
@@ -75,7 +75,7 @@ def generate_runs(
 
         # Calculating simulation results and error
         with open(os.path.join(runDirPath, 'sim_output'), 'w') as sim_output_file:
-            for i in range(num_simulations):
+            for i in range(sim_state["completed_sims"]):
                 log_file_path = os.path.join(runDirPath, f"{i + 1}", f"sim_{i + 1}.log")
                 sim_results = get_sim_results(open(log_file_path, "r").read())
 
@@ -86,6 +86,10 @@ def generate_runs(
                 error_data = calculate_sim_error(sim_output_lines=sim_output_file.readlines())
                 all_result_file.write("\n".join(error_data))
 
+        # dump final simulation state to log file called sim_state_file
+        with open(spiceDir + "/sim_state_file.txt", 'w') as sim_state_file:
+            json.dump(sim_state, sim_state_file)
+
     return runDirPath
 
 def matchNetlistCell(cell_instantiation):
@@ -169,4 +173,4 @@ def update_netlist(srcNetlist, dstNetlist, simMode):
 """
         netlist = netlist.replace(toplevel_pinout.split(" ", 2)[2], standardized_pinout)
     with open(dstNetlist, "w") as wf:
-        wf.write(netlist)
+        wf.write(netlist)

tests/common_api/test_simulation.py

@@ -60,7 +60,7 @@ def run_before_and_after_tests():
 		rmtree(RUNS_DIR)
 
 def test_simulations():
-	num_runs = run_simulations(
+	sim_state = run_simulations(
 		parameters=PARAMS,
 		platform = "",
 		simulation_dir = TEST_SIMULATION_DIR,
@@ -69,7 +69,7 @@ def test_simulations():
 	)
 
 	# Check if the correct number of configurations are generated
-	assert num_runs == EXPECTED_NUM_CONFIGS, "The number of runs does not match the expected number."
+	assert sim_state["completed_sims"] == EXPECTED_NUM_CONFIGS, "The number of runs does not match the expected number."
 	assert len(os.listdir(RUNS_DIR)) == EXPECTED_NUM_CONFIGS, "The number of generated configurations does not match the expected number."
 
 
@@ -99,4 +99,4 @@ def test_simulations():
 			irms_value = get_value(log_file_text, 'irms')
 
 			assert vrms_value != "NOT_FOUND", f"`vrms` value not found in the simulation output for config #{i}"
-			assert irms_value != "NOT_FOUND", f"`irms` value not found in the simulation output for config #{i}"
+			assert irms_value != "NOT_FOUND", f"`irms` value not found in the simulation output for config #{i}"