From 174b1eb057ec945829e94f4fc258d6cbe790aa06 Mon Sep 17 00:00:00 2001 From: B-4pt <48515764+B-apt@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:48:29 +0200 Subject: [PATCH 1/3] Fix .cup file parser and update README --- .gitignore | 4 + README.md | 41 ++++++++++ lib/xcsoar/mapgen/server/server.py | 12 +-- lib/xcsoar/mapgen/waypoints/seeyou_reader.py | 86 ++++++++++++-------- 4 files changed, 106 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index d12e78e..36edb9a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,7 @@ jobs bin/shptree bin/gdalwarp bin/ogr2ogr +/bin/ +/lib/python3.12 +pyvenv.cfg +lib64 \ No newline at end of file diff --git a/README.md b/README.md index e0b986d..1cb0a73 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,24 @@ These are named volumes inside your docker service. This directory caches all the data from the data repository. WARNING: This volume can take up a lot of space (100GB). +Note: To mount those volumes into a local directory to keep them if the containers are dropped, update the docker compose (change the path as needed): + +```bash +volumes: + mapgen-data: + driver: local + driver_opts: + type: none + o: bind + device: /home/user/xcsoar_mapgen/mapgen-data + mapgen-jobs: + driver: local + driver_opts: + type: none + o: bind + device: /home/user/xcsoar_mapgen/mapgen-jobs +``` + ### Ports ```bash @@ -83,3 +101,26 @@ docker-compose build \ ```bash docker-compose up -d ``` + +### Mounting the source files into the containers + +To speed up the development process, it's possible to mount the folders with the Python sources into the container, which allow automatic reload by CherryPy. +Update the docker-compose: + +```bash +... +services: + mapgen-frontend: +... + volumes: + - mapgen-jobs:/opt/mapgen/jobs + - ./lib:/opt/mapgen/lib + - ./bin:/opt/mapgen/bin + mapgen-worker: + ... + volumes: + - mapgen-jobs:/opt/mapgen/jobs + - mapgen-data:/opt/mapgen/data + - ./lib:/opt/mapgen/lib + - ./bin:/opt/mapgen/bin +``` diff --git a/lib/xcsoar/mapgen/server/server.py b/lib/xcsoar/mapgen/server/server.py index 94666b3..f7ebcdd 100644 --- a/lib/xcsoar/mapgen/server/server.py +++ b/lib/xcsoar/mapgen/server/server.py @@ -83,21 +83,23 @@ def index(self, **params): try: filename = waypoint_file.filename.lower() - if not filename.endswith(".dat") and ( - filename.endswith(".dat") or not filename.endswith(".cup") - ): + + if not (filename.endswith(".dat") or filename.endswith(".cup")): raise RuntimeError( "Waypoint file {} has an unsupported format.".format( waypoint_file.filename ) ) + + waypoint_file.file.seek(0) + desc.bounds = parse_waypoint_file( waypoint_file.filename, waypoint_file.file ).get_bounds() desc.waypoint_file = ( "waypoints.cup" if filename.endswith(".cup") else "waypoints.dat" ) - except: + except Exception as e: return view.render( error="Unsupported waypoint file " + waypoint_file.filename ) | HTMLFormFiller(data=params) @@ -134,7 +136,7 @@ def index(self, **params): if desc.waypoint_file: waypoint_file.file.seek(0) - f = open(job.file_path(desc.waypoint_file), "w") + f = open(job.file_path(desc.waypoint_file), "wb") try: shutil.copyfileobj(fsrc=waypoint_file.file, fdst=f, length=1024 * 64) finally: diff --git a/lib/xcsoar/mapgen/waypoints/seeyou_reader.py b/lib/xcsoar/mapgen/waypoints/seeyou_reader.py index f69564a..ded83f0 100644 --- a/lib/xcsoar/mapgen/waypoints/seeyou_reader.py +++ b/lib/xcsoar/mapgen/waypoints/seeyou_reader.py @@ -76,59 +76,81 @@ def __parse_length(str): def parse_seeyou_waypoints(lines, bounds=None): waypoint_list = WaypointList() - first = True - for line in lines: - if first: - first = False - continue + header = None + columns = {} - line = line.strip() - if line == "name,code,country,lat,lon,elev,style,rwdir,rwlen,freq,desc": - continue + for raw_line in lines: + if isinstance(raw_line, bytes): + line = raw_line.decode('utf-8', errors='ignore').strip() + else: + line = raw_line.strip() - if line == "" or line.startswith("*"): + if not line or line.startswith("*"): continue if line == "-----Related Tasks-----": break + # Parse CSV fields = [] - line = __CSVLine(line) - while line.has_next(): - fields.append(next(line)) - - if len(fields) < 6: + csv_line = __CSVLine(line) + while csv_line.has_next(): + fields.append(next(csv_line)) + + # First non-comment line is the header + if header is None: + header = [f.strip().lower() for f in fields] + columns = {name: idx for idx, name in enumerate(header)} continue - lat = __parse_coordinate(fields[3]) - if bounds and (lat > bounds.top or lat < bounds.bottom): - continue + def get(name, default=""): + idx = columns.get(name) + if idx is None or idx >= len(fields): + return default + return fields[idx] - lon = __parse_coordinate(fields[4]) - if bounds and (lon > bounds.right or lon < bounds.left): + try: + lat = __parse_coordinate(get("lat")) + lon = __parse_coordinate(get("lon")) + except Exception: continue + if bounds: + if lat > bounds.top or lat < bounds.bottom: + continue + if lon > bounds.right or lon < bounds.left: + continue + wp = Waypoint() wp.lat = lat wp.lon = lon - wp.altitude = __parse_altitude(fields[5]) - wp.name = fields[0].strip() - wp.country_code = fields[2].strip() - if len(fields) > 6 and len(fields[6]) > 0: - wp.cup_type = int(fields[6]) + wp.name = get("name").strip() + wp.country_code = get("country").strip() + + elev = get("elev") + if elev: + wp.altitude = __parse_altitude(elev) + + style = get("style") + if style: + wp.cup_type = int(style) - if len(fields) > 7 and len(fields[7]) > 0: - wp.runway_dir = int(fields[7]) + rwdir = get("rwdir") + if rwdir: + wp.runway_dir = int(rwdir) - if len(fields) > 8 and len(fields[8]) > 0: - wp.runway_len = __parse_length(fields[8]) + rwlen = get("rwlen") + if rwlen: + wp.runway_len = __parse_length(rwlen) - if len(fields) > 9 and len(fields[9]) > 0: - wp.freq = float(fields[9]) + freq = get("freq") + if freq: + wp.freq = float(freq) - if len(fields) > 10 and len(fields[10]) > 0: - wp.comment = fields[10].strip() + desc = get("desc") + if desc: + wp.comment = desc.strip() waypoint_list.append(wp) From 44a8f4f8747ecbb1d9c48531e503c0188228c77c Mon Sep 17 00:00:00 2001 From: B-4pt <48515764+B-apt@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:01:22 +0200 Subject: [PATCH 2/3] Fix PR comments on gitignore and decoding cup file in UTF8 or cp1252 --- .gitignore | 1 - lib/xcsoar/mapgen/waypoints/seeyou_reader.py | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 36edb9a..3c265af 100644 --- a/.gitignore +++ b/.gitignore @@ -15,7 +15,6 @@ jobs bin/shptree bin/gdalwarp bin/ogr2ogr -/bin/ /lib/python3.12 pyvenv.cfg lib64 \ No newline at end of file diff --git a/lib/xcsoar/mapgen/waypoints/seeyou_reader.py b/lib/xcsoar/mapgen/waypoints/seeyou_reader.py index ded83f0..bc58628 100644 --- a/lib/xcsoar/mapgen/waypoints/seeyou_reader.py +++ b/lib/xcsoar/mapgen/waypoints/seeyou_reader.py @@ -81,7 +81,11 @@ def parse_seeyou_waypoints(lines, bounds=None): for raw_line in lines: if isinstance(raw_line, bytes): - line = raw_line.decode('utf-8', errors='ignore').strip() + try: + line = raw_line.decode('utf-8') + except UnicodeDecodeError: + line = raw_line.decode('windows-1252') + line = line.strip() else: line = raw_line.strip() From 1432e583fcc69c43e9f37d275f77a7ce57fe411f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:04:37 +0000 Subject: [PATCH 3/3] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .gitignore | 2 +- lib/xcsoar/mapgen/waypoints/seeyou_reader.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 3c265af..3450a19 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,4 @@ bin/gdalwarp bin/ogr2ogr /lib/python3.12 pyvenv.cfg -lib64 \ No newline at end of file +lib64 diff --git a/lib/xcsoar/mapgen/waypoints/seeyou_reader.py b/lib/xcsoar/mapgen/waypoints/seeyou_reader.py index bc58628..3469df2 100644 --- a/lib/xcsoar/mapgen/waypoints/seeyou_reader.py +++ b/lib/xcsoar/mapgen/waypoints/seeyou_reader.py @@ -82,9 +82,9 @@ def parse_seeyou_waypoints(lines, bounds=None): for raw_line in lines: if isinstance(raw_line, bytes): try: - line = raw_line.decode('utf-8') + line = raw_line.decode("utf-8") except UnicodeDecodeError: - line = raw_line.decode('windows-1252') + line = raw_line.decode("windows-1252") line = line.strip() else: line = raw_line.strip()