Merge pull request #16 from rleyton/pdf-results

Pdf results

Pipfile

@@ -4,14 +4,17 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-pandas = "*"
+pandas = "<2"
 gspread = "*"
 python-dotenv = "*"
 black = "*"
 openpyxl = "*"
 tabulate = "*"
 pretty-html-table = "*"
 jinja2 = "*"
+markdown = "*"
+pdfkit = "*"
+pypdf = "*"
 
 [dev-packages]
 autopep8 = "*"

README.md

@@ -7,15 +7,18 @@
 ## Provisional results
 
 * _Provisional results reflect team submissions received by **23rd November, 7:30pm**_ - Please direct queries/issues to contact@westleague.org.uk
+* PDF results generated
+  * A single all-events PDF is [available here](https://github.com/rleyton/westleague/tree/main/results/provisional/2023-24/1/pdf/RESULTS.pdf)
+  * Hopefully this will help make the results a bit more digvetible
 * Website write-up will follow this weekend (25/26th) November
-  * Busy work week I'm afraid! ^RL
+  * Busy week work, and personally, I'm afraid! ^RL
 * [Event #1, Kilmarnock (18th November, 2023)](https://results.westleague.org.uk/results/provisional/2023-24/1/html/)
   * [Provisional Race results](https://results.westleague.org.uk/results/provisional/2023-24/1/html/) - See below for known issues
   * **4 Teams still need to submit results** - [List of teams here](https://github.com/rleyton/westleague/blob/main/results/provisional/2023-24/1/markdown/missingTeamSubmissions.md), [please submit via website here](https://westleague.org.uk/results/submission/)
   * Hosted by [Kilmarnock H&AC](http://www.kilmarnockharriers.com/)
   * [Attendance: 441](./results/provisional/2023-24/1/meta.json)
   * [Thanks to the 20 volunteers](./results/provisional/2023-24/1/html/volunteers.html)
-  * Data variations: [CSV](https://github.com/rleyton/westleague/tree/main/results/provisional/2023-24/1), [markdown](https://github.com/rleyton/westleague/tree/main/results/provisional/2023-24/1/markdown/)
+  * Data variations: [CSV](https://github.com/rleyton/westleague/tree/main/results/provisional/2023-24/1), [markdown](https://github.com/rleyton/westleague/tree/main/results/provisional/2023-24/1/markdown/), [PDF](https://github.com/rleyton/westleague/tree/main/results/provisional/2023-24/1/pdf/)
 
 
 ### Known results issues

tools/club-submissions-to-csv.py

@@ -18,7 +18,6 @@
 
 clubData = {}
 for club, filename in clubSubmissions:
-
     try:
         sheet = pd.read_excel(
             io=DATA_DIR + "/" + CLUB_SUBMISSIONS + "/" + filename,

tools/process-results.py

@@ -1,5 +1,4 @@
 #!env python
-
 import logging
 from src.adapter_sheets import load_volunteers, load_results
 from src.utils_consts import (
@@ -11,7 +10,9 @@
     GENDER_COMPETITION_MAP,
     HTML_DIR,
     MARKDOWN_DIR,
+    PDF_DIR,
 )
+from src.adapter_pdf import generate_pdf, combined_pdf, generate_single_pdf
 from src.utils_functions import fetch_events_from_dir, fetch_volunteers_from_dir
 from src.adapter_gender import gender_process
 from src.adapter_team import process_teams, load_team_submissions
@@ -24,13 +25,11 @@
 from src.adapter_times import adjust_times
 from src.adapter_places import adjust_places, process_final_results
 from src.adapter_points import calculate_competition_points
-from src.adapter_pretty_html import render
 from src.adapter_json import json_write
 from src.adapter_format import export_results, get_html
 from src.adapter_clubs import load_clubs
 import pandas as pd
 import pathlib
-from pretty_html_table import build_table
 from src.utils_config import DATA_DIR, ADJUSTMENTS_DIR, TEAMS, GENDERS, RESULTS_DIR
 
 logging.basicConfig(level=logging.DEBUG)
@@ -41,12 +40,14 @@
 pathlib.Path(RESULTS_DIR).mkdir(parents=True, exist_ok=True)
 pathlib.Path(RESULTS_DIR + MARKDOWN_DIR).mkdir(parents=True, exist_ok=True)
 pathlib.Path(RESULTS_DIR + HTML_DIR).mkdir(parents=True, exist_ok=True)
+pathlib.Path(RESULTS_DIR + PDF_DIR).mkdir(parents=True, exist_ok=True)
 
 
 teams = load_clubs()
 genders = pd.read_csv(GENDERS, index_col="shortcode")
 
 results = {}
+resultsHTML = {}
 leagueResults = {}
 teamResults = {}
 ageCatResults = {}
@@ -55,6 +56,8 @@
 index = []
 eventTotalParticipants = 0
 totalGenderParticipants = {}
+pdfs = []
+
 logging.info(f"Processing: {DATA_DIR}")
 
 # for each event we have files for
@@ -120,7 +123,7 @@
         get_missing_teams(results=results[event], submissions=clubSubmissions)
     )
     if len(eventMissingTeams) > 0:
-        logging.warning(f"Missing event submissions for {event}: {eventMissingTeams}")
+        logging.warning(f"Event submissions for {event}: {eventMissingTeams}")
         missingTeams = missingTeams.union(eventMissingTeams)
 
     # Now we have a results DataFrame, so process the competition results
@@ -151,6 +154,7 @@
     # ageCategory specific results
     if ageCatResults[event] is not None:
         for gender in ageCatResults[event]:
+            resultsHTML[gender] = {}
             for competition in ageCatResults[event][gender]:
                 # only write files if any data
                 if len(ageCatResults[event][gender][competition]) > 0:
@@ -162,6 +166,10 @@
                         base_file_name=baseFileName,
                         suffix=".agecat.results",
                     )
+                    resultsHTML[gender][competition] = (
+                        RESULTS_DIR + "/html/" + get_html(ageCatResultsPage)
+                    )
+
                     index.append(
                         {
                             "event": event,
@@ -176,17 +184,18 @@
     if teamResults[event] is not None:
         for gender in teamResults[event]:
             if gender not in totalGenderParticipants:
-                totalGenderParticipants[gender]=0
+                totalGenderParticipants[gender] = 0
 
             for competition in teamResults[event][gender]:
-
                 if competition not in eventMeta:
                     eventMeta[competition] = {}
 
                 eventMeta[competition][gender] = racemeta[competition][gender]
 
                 if competition != "OVERALL":
-                    totalGenderParticipants[gender]+=racemeta[competition][gender]["participants"]
+                    totalGenderParticipants[gender] += racemeta[competition][gender][
+                        "participants"
+                    ]
 
                 baseFileName = (
                     event + "." + competition + "." + GENDER_COMPETITION_MAP[gender]
@@ -198,6 +207,15 @@
                     suffix=".team.results",
                 )
 
+                pdfs.append(
+                    generate_pdf(
+                        competition=competition,
+                        gender=gender,
+                        resultshtml=resultsHTML[gender][competition],
+                        teamhtml=RESULTS_DIR + "/html/" + get_html(resultPages),
+                    )
+                )
+
                 index.append(
                     {
                         "event": event,
@@ -216,7 +234,13 @@
 
 # TODO: This all needs refactoring/tidying up
 json_write(
-    object={"races": eventMeta, "attendance": {"total": eventTotalParticipants,"gender":totalGenderParticipants}},
+    object={
+        "races": eventMeta,
+        "attendance": {
+            "total": eventTotalParticipants,
+            "gender": totalGenderParticipants,
+        },
+    },
     filename=RESULTS_DIR + "/" + "meta.json",
 )
 
@@ -227,7 +251,7 @@
     other=teams, on="team", lsuffix="missing", rsuffix="teamDetails"
 )
 
-results = export_results(
+missing_teams = export_results(
     results=theMissingTeams, base_file_name="", suffix="missingTeamSubmissions"
 )
 
@@ -236,7 +260,7 @@
         "event": "n/a",
         "competition": "n/a",
         "gender": "n/a",
-        "results": get_html(results),
+        "results": get_html(missing_teams),
     }
 )
 
@@ -256,7 +280,22 @@
             "results": get_html(vols),
         }
     )
+    pdfs.append(
+        generate_single_pdf(
+            html=RESULTS_DIR + HTML_DIR + "/" + get_html(vols),
+            filename="volunteers",
+            summary="Volunteers",
+        )
+    )
 
+pdfs.insert(
+    0,
+    generate_single_pdf(
+        html=RESULTS_DIR + HTML_DIR + "/" + get_html(missing_teams),
+        filename="missingTeams",
+        summary="Team submissions still pending",
+    ),
+)
 
 # Render a basic HTML index
 
@@ -276,4 +315,30 @@ def make_clickable(val):
 )
 
 
+summary = """
+<h1>Summary</h1>
+<p>Combined results PDF. Please see <a href="https://westleague.org.uk">westleague.org.uk</a></p>
+<p>Latest results/status at <a href="https://results.westleague.org.uk">results.westleague.org.uk</a></p>
+<p>Note results are provisional until all team results received.</p>
+<h1>Results</h1>
+<p>Results for all events in following order<p>
+<ul>
+<li>Still pending team submissions</li>
+<li>U11: Male, Male Teams, Female, Female Teams</li>
+<li>U13: Male, Male Teams, Female, Female Teams</li>
+<li>U15: Male, Male Teams, Female, Female Teams</li>
+<li>U17: Male, Male Teams, Female, Female Teams</li>
+<li>U20: Male, Male Teams, Female, Female Teams</li>
+<li>Senior: Male, Male Teams, Female, Female Teams</li>
+<li>Master: Male, Male Teams, Female, Female Teams</li>
+<li>Overall: Male, Male Teams, Female, Female Teams</li>
+<li>Volunteers
+</ul>
+<p>
+"""
+combined_pdf(
+    pdf_list=pdfs, target=RESULTS_DIR + PDF_DIR + "/RESULTS.pdf", summary=summary
+)
+
+
 logging.info("Finished")

tools/src/adapter_format.py

@@ -3,13 +3,41 @@
 from .utils_consts import MARKDOWN_DIR, HTML_DIR
 from .adapter_pretty_html import render
 import pathlib
+import markdown
+from markdown import Markdown
+from io import StringIO
+
+
+def unmark_element(element, stream=None):
+    if stream is None:
+        stream = StringIO()
+    if element.text:
+        stream.write(element.text)
+    for sub in element:
+        unmark_element(sub, stream)
+    if element.tail:
+        stream.write(element.tail)
+    return stream.getvalue()
+
+
+# patching Markdown
+Markdown.output_formats["plain"] = unmark_element
+__md = Markdown(output_format="plain")
+__md.stripTopLevelTags = False
+
+
+def unmark(text):
+    return __md.convert(text)
 
 
 def check_dirs_exist(dir: str):
     pathlib.Path(dir + MARKDOWN_DIR).mkdir(parents=True, exist_ok=True)
     pathlib.Path(dir + HTML_DIR).mkdir(parents=True, exist_ok=True)
 
 
+# markdown = Markdown()
+
+
 def export_results(
     results=None,
     results_dir: str = RESULTS_DIR,
@@ -31,6 +59,7 @@ def export_results(
     }
     results.to_csv(files["csv"], index=index)
     results.to_markdown(files["markdown"], index=index)
+    # markdown.markdownFromFile(input=files["markdown"],output=files["html"],encoding="utf8",extensions=['markdown.extensions.tables'])
     render(df=results, style="blue_light", filename=files["html"])
 
     return files

tools/src/adapter_json.py

@@ -2,7 +2,6 @@
 
 
 def json_write(object=None, filename: str = None):
-
     with open(filename, "w") as fh:
         json.dump(object, fh, indent=6)
 

tools/src/adapter_pdf.py

@@ -0,0 +1,79 @@
+import pdfkit
+import tempfile
+from pypdf import PdfMerger
+
+from .utils_config import RESULTS_DIR
+from .utils_consts import PDF_DIR
+
+
+def get_options(header):
+    return {
+        "page-size": "A4",
+        "margin-top": "0.9in",
+        "margin-right": "0.9in",
+        "margin-bottom": "0.9in",
+        "margin-left": "0.9in",
+        "encoding": "UTF-8",
+        "header-center": f"{header}",
+        "custom-header": [("Accept-Encoding", "gzip")],
+        "no-outline": None,
+    }
+
+
+def do_html_to_pdf(file, html, summary):
+    if type(file) is str:
+        filename = file
+    else:
+        filename = file.name
+
+    pdfkit.from_file(html, filename, options=get_options(header=summary))
+
+
+def generate_pdf(competition, gender, resultshtml, teamhtml, summary=None):
+    output_file = RESULTS_DIR + PDF_DIR + f"/{competition}-{gender}.pdf"
+    result_file = tempfile.NamedTemporaryFile(
+        suffix=".pdf",
+    )
+    team_file = tempfile.NamedTemporaryFile(
+        suffix=".pdf",
+    )
+
+    do_html_to_pdf(
+        file=result_file,
+        html=resultshtml,
+        summary=f"{competition} {gender} - race results",
+    )
+    do_html_to_pdf(
+        file=team_file, html=teamhtml, summary=f"{competition} {gender} - team results"
+    )
+
+    merger = PdfMerger()
+    for file in [result_file, team_file]:
+        merger.append(file.name)
+
+    merger.write(output_file)
+    merger.close()
+
+    return output_file
+
+
+def generate_single_pdf(filename, html, summary=None):
+    output_file = RESULTS_DIR + PDF_DIR + "/" + filename + ".pdf"
+
+    do_html_to_pdf(file=output_file, html=html, summary=summary)
+
+    return output_file
+
+
+def combined_pdf(pdf_list, summary, target):
+    merger = PdfMerger()
+    summary_file = tempfile.NamedTemporaryFile(suffix=".pdf")
+
+    if summary is not None:
+        pdfkit.from_string(input=summary, output_path=summary_file.name)
+        merger.append(summary_file.name)
+    for pdf in pdf_list:
+        merger.append(pdf)
+
+    merger.write(target)
+    merger.close()