diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 00d3392..e429383 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -1,14 +1,18 @@ -name: Deploy API +name: CI on: push: branches: - main + pull_request: workflow_dispatch: jobs: - deploy: + ci: runs-on: ubuntu-latest + env: + USE_SQLITE: "true" + DJANGO_SECRET_KEY: ci-only-secret-key steps: - name: Checkout code uses: actions/checkout@v4 @@ -28,11 +32,10 @@ jobs: working-directory: ./bedelia run: | python manage.py check - python manage.py test || echo "No tests configured yet" + python manage.py test - name: Lint with flake8 working-directory: ./bedelia run: | pip install flake8 - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics || true - + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics diff --git a/QUICKSTART.md b/QUICKSTART.md index 8e46dce..8e480a6 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -2,33 +2,39 @@ Guía rápida para poner en marcha los modelos y cargar datos en menos de 5 minutos. -## 🚀 Inicio Rápido (3 comandos) +Todos los comandos se ejecutan desde el directorio `bedelia/` (donde está `manage.py`). +Con `USE_SQLITE=true` no hace falta PostgreSQL. + +## 🚀 Inicio Rápido ```bash -# 1. Crear y aplicar migraciones -python manage.py makemigrations -python manage.py migrate +cd bedelia + +# 1. Aplicar migraciones +USE_SQLITE=true python manage.py migrate # 2. Verificar con dry-run (opcional pero recomendado) -python manage.py load_bedelia_data --dry-run +USE_SQLITE=true python manage.py load_bedelia_data --dry-run # 3. Cargar datos -python manage.py load_bedelia_data --clear --verbose +USE_SQLITE=true python manage.py load_bedelia_data --clear --verbose ``` ¡Listo! 🎉 --- -## 📋 Lo que acabas de crear +## 📋 Modelos principales ``` -Estructura de Base de Datos: -├── carreras (45 registros aprox) -├── cursos (1,200+ registros) -├── previas (5,000+ nodos de árbol) -├── items_previa (10,000+ items) -└── posprevias (8,000+ relaciones) +api/models.py: +├── PlanEstudio (carrera + año del plan) +├── Materia (curso con código y créditos) +├── PlanMateria (materia dentro de un plan) +├── UnidadAprobable (curso / examen / módulo aprobable) +├── PreviaNodo (árbol lógico de previas: ALL / ANY / NOT / LEAF) +├── PreviaItem (condición concreta de un nodo LEAF) +└── PosPreviaItem (dependencias inversas: qué requiere esta materia) ``` --- @@ -36,105 +42,72 @@ Estructura de Base de Datos: ## 🧪 Verificar que funciona ```bash -python manage.py shell +USE_SQLITE=true python manage.py shell ``` ```python -from api.models import Carrera, Curso, Previa +from api.models import PlanEstudio, Materia, PreviaNodo # Ver totales -print(f"Carreras: {Carrera.objects.count()}") -print(f"Cursos: {Curso.objects.count()}") -print(f"Previas: {Previa.objects.count()}") +print(f"Planes de estudio: {PlanEstudio.objects.count()}") +print(f"Materias: {Materia.objects.count()}") +print(f"Nodos de previas: {PreviaNodo.objects.count()}") # Ver ejemplo -curso = Curso.objects.first() -print(f"\nEjemplo: {curso}") -print(f"Créditos: {curso.creditos}") -print(f"Carreras: {curso.carrera.count()}") -``` - -**Salida esperada:** -``` -Carreras: 45 -Cursos: 1234 -Previas: 5678 - -Ejemplo: 1267 - TALLER REPR. Y COM. GRAFICA -Créditos: 5 -Carreras: 2 +materia = Materia.objects.first() +print(f"\nEjemplo: {materia}") +print(f"Créditos: {materia.creditos}") ``` --- -## 📚 Consultas Útiles +## 📚 Consultas útiles -### Ver todas las carreras +### Ver planes de estudio ```python -from api.models import Carrera -for c in Carrera.objects.all()[:10]: - print(f"- {c.nombre} ({c.anio_plan})") +from api.models import PlanEstudio +for p in PlanEstudio.objects.all()[:10]: + print(f"- {p.nombre_carrera} ({p.anio})") ``` -### Buscar un curso +### Buscar una materia ```python -from api.models import Curso -curso = Curso.objects.filter(codigo_curso="1144").first() -print(curso.nombre_curso) -print(f"Créditos: {curso.creditos}") +from api.models import Materia +materia = Materia.objects.filter(codigo="1144").first() +print(materia.nombre, materia.creditos) ``` -### Ver árbol de previas +### Ver nodos raíz del árbol de previas ```python -curso = Curso.objects.get(codigo_curso="1144") -previas_raiz = curso.previas.filter(padre__isnull=True) -for previa in previas_raiz: - print(f"Tipo: {previa.tipo}") - print(f"Título: {previa.titulo}") +from api.models import PreviaNodo +for nodo in PreviaNodo.objects.filter(padre__isnull=True)[:5]: + print(f"Tipo: {nodo.tipo} - {nodo.descripcion}") ``` ### Ver posprevias (qué materias requieren este curso) ```python -curso = Curso.objects.get(codigo_curso="1061") -posprevias = curso.posprevias.all()[:5] -for p in posprevias: - print(f"- {p.materia_nombre}") +from api.models import Materia +materia = Materia.objects.filter(codigo="1061").first() +for p in materia.posprevias.all()[:5]: + print(f"- {p.plan_estudio}: {p.descripcion}") ``` --- ## 🎯 Datos Clave -### Estructura de Archivos JSON +### Estructura de Archivos JSON (en `../data/`) - ✅ **vigentes**: TODOS los cursos activos -- ✅ **credits**: Créditos de los cursos +- ✅ **credits**: Créditos de los cursos - ⚠️ **previas**: Solo cursos CON requisitos (NO todos) - ✅ **posprevias**: Dependencias inversas ### Es Normal -✅ Que haya cursos sin previas -✅ Que el proceso tome 3-5 minutos -✅ Ver advertencias sobre cursos no encontrados - -### NO es Normal - -❌ Que todos los cursos tengan previas -❌ Errores de base de datos -❌ Que falten archivos JSON - ---- - -## 🔄 Actualizar Datos - -```bash -# Borrar todo y recargar -python manage.py load_bedelia_data --clear - -# Actualización incremental (solo nuevos/modificados) -python manage.py load_bedelia_data -``` +✅ Que haya cursos sin previas +✅ Que el proceso tome 3-5 minutos +✅ Ver advertencias sobre cursos no encontrados --- @@ -142,32 +115,18 @@ python manage.py load_bedelia_data ### "Archivo no encontrado" ```bash -# Verificar que los archivos existan -ls data/*.json +# Los JSON viven en data/ en la raíz del repo +ls ../data/*.json ``` -### "Out of memory" -- Usar PostgreSQL en lugar de SQLite -- Aumentar memoria disponible -- Cerrar otras aplicaciones - ### "Integrity error" ```bash # Limpiar y reintentar -python manage.py load_bedelia_data --clear +USE_SQLITE=true python manage.py load_bedelia_data --clear ``` --- -## 📖 Más Información - -- [SETUP_GUIDE.md](SETUP_GUIDE.md) - Guía completa paso a paso -- [CHANGELOG_MODELS.md](CHANGELOG_MODELS.md) - Detalles de implementación -- [bedelia/api/ESTRUCTURA_MODELOS.md](bedelia/api/ESTRUCTURA_MODELOS.md) - Documentación de modelos -- [bedelia/api/management/commands/README_load_bedelia_data.md](bedelia/api/management/commands/README_load_bedelia_data.md) - Documentación del comando - ---- - ## 🆘 Ayuda Rápida ```bash @@ -175,15 +134,11 @@ python manage.py load_bedelia_data --clear python manage.py load_bedelia_data --help # Probar sin guardar -python manage.py load_bedelia_data --dry-run --verbose - -# Ver qué se cargó -python manage.py shell < bedelia/api/management/commands/verify_data.py +USE_SQLITE=true python manage.py load_bedelia_data --dry-run --verbose ``` --- **¡Ya estás listo para usar la API! 🚀** -Siguiente paso: Crear serializers y endpoints REST → Ver [TODO.md](TODO.md) - +Con el servidor corriendo (`python manage.py runserver`), la documentación interactiva está en `/api/docs/` y hay una colección de Postman en la raíz del repo. diff --git a/README.md b/README.md index 6da19cc..1eb2074 100644 --- a/README.md +++ b/README.md @@ -1,125 +1,85 @@ -# Bedelías API Scraper +# Bedelías API -An improved web scraper for extracting academic data from the Bedelías system at Universidad de la República (Uruguay). +![CI](https://github.com/moura-code/bedelia_api/actions/workflows/deploy.yml/badge.svg) -## Features +Structured academic data from **Bedelías**, the student records system of Universidad de la República (Uruguay), which has no public API. -- **Modular Design**: Separated page classes for better code organization -- **Improved Error Handling**: Better logging and error management -- **Flexible Configuration**: Environment variable support with sensible defaults -- **Browser Support**: Works with Firefox and Chrome browsers -- **Selective Extraction**: Choose to extract previas, posprevias, or both +Two components: -## Project Structure +1. **Scraper** (`scraper/`) — a modular Selenium scraper that logs into Bedelías and extracts course prerequisites (*previas*), reverse dependencies (*posprevias*), credits and active courses (*vigentes*). +2. **REST API** (`bedelia/`) — a Django REST Framework API that serves the scraped data (careers, courses, prerequisite trees) with filtering and OpenAPI docs. + +## Project structure ``` -scraper/ -├── main.py # Main Bedelias class and entry point -├── scraper.py # Base Scraper class with common functionality -├── example_usage.py # Usage examples -├── pages/ # Page-specific scrapers -│ ├── login.py # Login functionality -│ ├── previas.py # Prerequisites extraction -│ └── posprevias.py # Post-prerequisites extraction -└── common/ - └── usetable.py # Table pagination utilities +├── scraper/ # Selenium scraper +│ ├── main.py # Bedelias orchestrator and entry point +│ ├── scraper.py # Base Scraper class +│ ├── config.py # Env-based configuration (ScraperConfig) +│ ├── pages/ # One extractor per Bedelías page +│ │ ├── login.py, previas.py, posprevias.py, credits.py, vigentes.py +│ └── common/ # Navigation and table-pagination helpers +├── bedelia/ # Django REST API +│ ├── api/ # Models, serializers, views, data-load command +│ └── config/ # Settings, URLs, exception handling +├── data/ # Scraped JSON backups +├── eligible_courses.py # Analysis: which courses can I take now? +└── get_libres.py # Analysis: courses that can be taken "libre" ``` -## Installation +## Scraper + +### Setup -1. Install required dependencies: ```bash pip install -r requirements.txt ``` -2. Set up your credentials in a `.env` file: +Create a `.env` file with your Bedelías credentials: + ```env DOCUMENTO=your_document_number CONTRASENA=your_password -BROWSER=firefox -DEBUG=False -EXTRACT_PREVIAS=True -EXTRACT_POSPREVIAS=True +BROWSER=firefox # or chrome +DEBUG=False # True shows the browser window +PAGES=previas,vigentes,posprevias,credits # optional, defaults to all ``` -## Usage - -### Basic Usage - -```python -from main import Bedelias - -# Initialize scraper -scraper = Bedelias( - username="your_document", - password="your_password", - browser="firefox", - debug=True # Set to False for headless mode -) +### Run -# Run complete extraction -scraper.run() +```bash +cd scraper +python main.py ``` -### Advanced Usage - -```python -# Extract only previas -scraper.run(extract_previas=True, extract_posprevias=False) +Extracted data is written as JSON backups (see `data/`). On failure, a `screenshot.png` of the browser is saved to help debugging. -# Extract only posprevias -scraper.run(extract_previas=False, extract_posprevias=True) +## REST API -# Use environment variables -scraper = Bedelias() # Will load from .env -scraper.run() +```bash +cd bedelia +pip install -r requirements.txt +USE_SQLITE=true python manage.py migrate +USE_SQLITE=true python manage.py load_bedelia_data --clear --verbose +USE_SQLITE=true python manage.py runserver ``` -### Command Line +Or with Docker (PostgreSQL included): ```bash -# Run with environment variables -python main.py - -# Run example -python example_usage.py - -# Run minimal example -python example_usage.py minimal +cd bedelia +docker compose up --build ``` -## Configuration Options - -| Environment Variable | Default | Description | -|---------------------|---------|-------------| -| `DOCUMENTO` | - | Your document number (required) | -| `CONTRASENA` | - | Your password (required) | -| `BROWSER` | `firefox` | Browser to use (`firefox` or `chrome`) | -| `DEBUG` | `False` | Enable debug mode (show browser) | -| `EXTRACT_PREVIAS` | `True` | Extract prerequisites data | -| `EXTRACT_POSPREVIAS` | `True` | Extract post-prerequisites data | - -## Output +See [QUICKSTART.md](QUICKSTART.md) for the data model and verification steps, and the Postman collection (`bedelia_previas_api.postman_collection.json`) for example requests. -The scraper generates JSON backup files: -- `previas_data_backup.json` - Prerequisites data -- `posprevias_data_backup.json` - Post-prerequisites data +### Configuration -## Improvements Made - -1. **Fixed Missing Functions**: Completed `build_driver()` and other incomplete functions -2. **Proper Class Integration**: Page classes now properly inherit from base Scraper class -3. **Fixed Imports**: Resolved circular import issues and missing dependencies -4. **Better Error Handling**: Improved logging and error management throughout -5. **Flexible API**: Added parameters to control what data to extract -6. **Documentation**: Added comprehensive documentation and examples - -## Error Handling - -The scraper includes comprehensive error handling: -- Network timeouts and connection issues -- Missing elements on pages -- Login failures -- Data extraction errors - -All errors are logged appropriately, and the browser is properly cleaned up even if errors occur. +| Environment variable | Default | Description | +|---------------------|---------|-------------| +| `USE_SQLITE` | `False` | Use SQLite instead of PostgreSQL | +| `DATABASE_URL` | — | Full database URL (takes precedence) | +| `DJANGO_SECRET_KEY` | dev-only fallback | Set a real key in production | +| `DEBUG` | `true` | Django debug mode | +| `ALLOWED_HOSTS` | `*` | Comma-separated hosts | +| `CORS_ALLOWED_ORIGINS` | allow all | Comma-separated origins | diff --git a/bedelia/api/management/commands/load_bedelia_data.py b/bedelia/api/management/commands/load_bedelia_data.py index b3a08c2..f360eee 100644 --- a/bedelia/api/management/commands/load_bedelia_data.py +++ b/bedelia/api/management/commands/load_bedelia_data.py @@ -855,10 +855,13 @@ def _process_course_previas(self, plan: PlanEstudio, course_key: str, course_dat self.stdout.write(self.style.WARNING(f' [#] {error_msg}')) return - plan_materia, _ = PlanMateria.objects.get_or_create( - plan=plan, - materia=materia - ) + if self.dry_run: + plan_materia = PlanMateria(plan=plan, materia=materia) + else: + plan_materia, _ = PlanMateria.objects.get_or_create( + plan=plan, + materia=materia + ) except Exception as e: error_msg = f"No se pudo obtener PlanMateria para {course_code}-{course_type}: {str(e)}" self.add_error( @@ -1250,6 +1253,10 @@ def process_posprevias(self, posprevias_data: Dict): self.stdout.write(f' [{idx}/{total_plans}] Procesando plan: {carrera_plan}') carrera_nombre, anio = self.parse_carrera_plan(carrera_plan) + + if not carrera_nombre or not anio: + continue + # Obtener plan try: plan = self.get_or_create_plan(carrera_nombre, anio) @@ -1317,10 +1324,13 @@ def _process_course_posprevias(self, source_plan: PlanEstudio, source_course_cod # Obtener o crear PlanMateria fuente try: - plan_materia_fuente, _ = PlanMateria.objects.get_or_create( - plan=source_plan, - materia=materia_fuente - ) + if self.dry_run: + plan_materia_fuente = PlanMateria(plan=source_plan, materia=materia_fuente) + else: + plan_materia_fuente, _ = PlanMateria.objects.get_or_create( + plan=source_plan, + materia=materia_fuente + ) except Exception as e: error_msg = f"Error obteniendo PlanMateria fuente para {source_course_code}: {str(e)}" self.add_error( diff --git a/bedelia/api/models.py b/bedelia/api/models.py index cef05bb..9a0127f 100644 --- a/bedelia/api/models.py +++ b/bedelia/api/models.py @@ -175,7 +175,7 @@ class PosPreviaItem(models.Model): fecha_modificacion = models.DateTimeField(auto_now=True) def __str__(self) -> str: - return f"{self.materia} :: {self.materia_dependiente} :: {self.descripcion}" + return f"{self.materia} :: {self.plan_estudio} :: {self.descripcion}" # ============================================================ # Unidades aprobables (Curso / Examen / etc.) diff --git a/bedelia/api/serializers/materias.py b/bedelia/api/serializers/materias.py index 30069ab..7939201 100644 --- a/bedelia/api/serializers/materias.py +++ b/bedelia/api/serializers/materias.py @@ -235,7 +235,8 @@ class PreviaNodoSerializer(serializers.ModelSerializer): padre_id = serializers.UUIDField(source='padre.id', read_only=True, allow_null=True) tipo_display = serializers.CharField(source='get_tipo_display', read_only=True) hijos_count = serializers.SerializerMethodField() - + items_count = serializers.SerializerMethodField() + class Meta: model = PreviaNodo fields = [ @@ -258,6 +259,11 @@ class Meta: def get_hijos_count(self, obj: PreviaNodo) -> int: """Obtener cantidad de nodos hijos.""" return obj.hijos.count() + + @extend_schema_field(serializers.IntegerField()) + def get_items_count(self, obj: PreviaNodo) -> int: + """Obtener cantidad de items del nodo.""" + return obj.items.count() class PreviaNodoTreeSerializer(serializers.ModelSerializer): diff --git a/bedelia/api/views/materias.py b/bedelia/api/views/materias.py index 5b9da54..06a4df8 100644 --- a/bedelia/api/views/materias.py +++ b/bedelia/api/views/materias.py @@ -228,54 +228,6 @@ class UnidadAprobableViewSet(viewsets.ReadOnlyModelViewSet): ordering = ['materia__codigo', 'tipo'] -@extend_schema_view( - list=extend_schema( - summary="Listar todos los nodos de requisitos", - description="Obtener una lista paginada de todos los nodos de requisitos (nodos del árbol de requisitos). Soporta filtrado por plan_materia, tipo, padre, opción root_only y estado activo.", - tags=["requisitos"], - parameters=[ - OpenApiParameter( - name='plan_materia', - type=str, - location=OpenApiParameter.QUERY, - description='Filtrar por ID de plan_materia (UUID)', - ), - OpenApiParameter( - name='tipo', - type=str, - location=OpenApiParameter.QUERY, - description='Filtrar por tipo: ALL, ANY, NOT, LEAF', - ), - OpenApiParameter( - name='padre', - type=str, - location=OpenApiParameter.QUERY, - description='Filtrar por ID de nodo padre (UUID)', - ), - OpenApiParameter( - name='root_only', - type=bool, - location=OpenApiParameter.QUERY, - description='Filtrar para mostrar solo nodos raíz (true/false)', - ), - OpenApiParameter( - name='activo', - type=bool, - location=OpenApiParameter.QUERY, - description='Filtrar por estado activo del plan y la materia de plan_materia (true/false)', - ), - ], - ), - retrieve=extend_schema( - summary="Obtener detalles de un nodo de requisito", - description="Obtener información detallada sobre un nodo de requisito específico. La estructura del árbol se devuelve unificada con el campo 'children' que contiene sub-nodos o items según el tipo.", - tags=["requisitos"], - ), -) - - - - @extend_schema_view( list=extend_schema( summary="Obtener previas (requisitos previos) para una PlanMateria", diff --git a/bedelia/config/settings.py b/bedelia/config/settings.py index d9289d3..e8b41e6 100644 --- a/bedelia/config/settings.py +++ b/bedelia/config/settings.py @@ -26,7 +26,8 @@ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY") +# The fallback is for local development only — set DJANGO_SECRET_KEY in production. +SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY", "django-insecure-dev-only-key") # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.environ.get("DEBUG", "true").lower() == "true" @@ -216,6 +217,7 @@ 'rest_framework.renderers.JSONRenderer', 'rest_framework.renderers.BrowsableAPIRenderer', ], + 'EXCEPTION_HANDLER': 'config.exceptions.exceptions.global_exception_handler', } # drf-spectacular (OpenAPI/Swagger) configuration @@ -258,8 +260,8 @@ ], 'ENUM_NAME_OVERRIDES': { 'UnidadAprobableTipoEnum': 'api.models.UnidadAprobable.Tipo', - 'RequisitoNodoTipoEnum': 'api.models.RequisitoNodo.Tipo', - 'RequisitoItemTipoEnum': 'api.models.RequisitoItem.TipoItem', + 'PreviaNodoTipoEnum': 'api.models.PreviaNodo.Tipo', + 'PreviaItemTipoEnum': 'api.models.PreviaItem.TipoItem', }, 'SWAGGER_UI_SETTINGS': { 'deepLinking': True, diff --git a/bedelia/requirements.txt b/bedelia/requirements.txt index 2951a0e..e63a5ee 100644 --- a/bedelia/requirements.txt +++ b/bedelia/requirements.txt @@ -6,3 +6,4 @@ drf-spectacular==0.27.2 python-dotenv==1.0.1 dj-database-url==2.3.0 gunicorn==23.0.0 +psycopg2-binary==2.9.10 diff --git a/get_libres.py b/get_libres.py index b2a60fe..ac82c68 100644 --- a/get_libres.py +++ b/get_libres.py @@ -151,8 +151,8 @@ def main(): if course.get("type_previas") != "Examen": continue # only look at exam-type entries - exam_code = course.get("code", "") - exam_name = course.get("name", "") + exam_code = course.get("code") or "" + exam_name = course.get("name") or "" requirements = course.get("requirements") vigentes_course = vigentes.get(exam_code) diff --git a/requirements.txt b/requirements.txt index f6de79c..2591c69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,4 +12,3 @@ dj-database-url>=2.2,<3.0 # Used for DATABASE_URL parsing (optional) whitenoise>=6.7,<7.0 djangorestframework-simplejwt>=5.3,<6.0 gunicorn>=22.0,<24.0 ; platform_system != "Windows" -python-dotenv==0.10.1 \ No newline at end of file diff --git a/scraper/common/navigation.py b/scraper/common/navigation.py index 6840ec5..4a0f3bc 100644 --- a/scraper/common/navigation.py +++ b/scraper/common/navigation.py @@ -72,7 +72,11 @@ def get_total_plan_sections(self): tbody = planes_div.find_element(By.CLASS_NAME, "ui-datatable-data") plan_rows = tbody.find_elements(By.TAG_NAME, "tr") for plan_row in plan_rows: + if "ui-datatable-empty-message" in (plan_row.get_attribute("class") or ""): + continue cells = plan_row.find_elements(By.TAG_NAME, "td") + if len(cells) < 3: + continue year = cells[0].text.strip() plan_name = row.find_element(By.XPATH, "./td[2]").text.strip() vigente = cells[2].get_attribute("innerHTML").strip() @@ -102,8 +106,12 @@ def open_plan_section(self, *, log_message: str, plan_name: str, plan_year: str, (By.XPATH, f'//*[text()="{plan_name}"]/preceding-sibling::td[1]') )) - # Click on the info icon for the row that has both the matching year AND "Si" in vigente - xpath = f'//tr[td[1][text()="{plan_year}"]][td[3][text()="Si"]]//i[contains(@class, "pi-info-circle") or contains(@class, "pi-calendar")]' + # Click on the info icon for the row that has both the matching year AND "Si"/"Sí" in vigente + xpath = ( + f'//tr[td[1][text()="{plan_year}"]]' + f'[td[3][text()="Si" or text()="Sí" or text()="si" or text()="sí"]]' + f'//i[contains(@class, "pi-info-circle") or contains(@class, "pi-calendar")]' + ) self.logger.info(f"Clicking plan: year={plan_year}, vigente=Si") if not second: diff --git a/scraper/common/usetable.py b/scraper/common/usetable.py index 61b944c..2b984a3 100644 --- a/scraper/common/usetable.py +++ b/scraper/common/usetable.py @@ -35,6 +35,10 @@ def get_total_pages(self) -> int: if element.is_displayed(): active = element break + if active is None: + self.logger.warning("No visible active paginator anchor; assuming a single page") + self.total_pages = 1 + return 1 total = int(active.text.strip()) self.logger.info(f"Total pages: {total}") diff --git a/scraper/main.py b/scraper/main.py index e4cd507..7e1a8c0 100644 --- a/scraper/main.py +++ b/scraper/main.py @@ -90,6 +90,7 @@ def __init__(self, config: ScraperConfig): self.previas_page = None self.posprevias_page = None self.credits_page = None + self.vigentes_page = None def start_driver(self): """Initialize the Selenium driver and page objects.""" @@ -231,6 +232,12 @@ def run(self): traceback.print_exc() else: self.logger.error("Scraping error: %s", exc) + if self.driver: + try: + self.driver.get_screenshot_as_file("screenshot.png") + self.logger.info("Saved failure screenshot to screenshot.png") + except Exception: + self.logger.warning("Could not capture failure screenshot") raise finally: self.logger.info("Cleaning up and closing browser...") @@ -250,9 +257,9 @@ def main(): scraper = Bedelias(config) try: scraper.run() - except Exception as exc: + except Exception: traceback.print_exc() - scraper.driver.get_screenshot_as_file("screenshot.png") + sys.exit(1) except KeyboardInterrupt: logger.info("Scraping interrupted by user") sys.exit(0) diff --git a/scraper/pages/credits.py b/scraper/pages/credits.py index 41c7951..53d4338 100644 --- a/scraper/pages/credits.py +++ b/scraper/pages/credits.py @@ -65,7 +65,11 @@ def _process_plan_with_retry(self, plan: str, year: str, max_retries: int = 3) - items = {} self.logger.info("Open all nodes") + expand_attempts = 0 while self.try_find_element((By.XPATH, '//*[@class="ui-tree-toggler ui-icon ui-icon-triangle-1-e"]')): + expand_attempts += 1 + if expand_attempts > 300: + raise Exception("Failed to expand all credit tree nodes") try: self.scroll_to_element_and_click(self.driver.find_element(By.XPATH, '//*[@class="ui-tree-toggler ui-icon ui-icon-triangle-1-e"]')) except Exception as e: @@ -74,10 +78,8 @@ def _process_plan_with_retry(self, plan: str, year: str, max_retries: int = 3) - self.logger.info("Searching for course materials...") materia_elements = self.driver.find_elements(By.XPATH, '//*[@data-nodetype="Materia"]') self.logger.info(f"Found {len(materia_elements)} course materials to process") - if self.try_find_element((By.XPATH, './/td/span[@title="Código - Nombre - Créd.aportado"]')): - x_path_title = './/td/span[@title="Código - Nombre - Créd.aportado"]' - else: - x_path_title = './/td/span[@title="Código - Nombre"]' + # Titles may be "Código - Nombre - Créd.aportado" or "Código - Nombre" per node + x_path_title = './/td/span[starts-with(@title, "Código - Nombre")]' for idx, li in enumerate(materia_elements, 1): self.logger.debug(f"Processing material {idx}/{len(materia_elements)}") # Get the span that holds the line "CODE - NAME - whatever" diff --git a/scraper/pages/posprevias.py b/scraper/pages/posprevias.py index 861943a..04f8eb4 100644 --- a/scraper/pages/posprevias.py +++ b/scraper/pages/posprevias.py @@ -1,8 +1,8 @@ import sys import os -from common.navigation import PlanSection sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from common.navigation import PlanSection from scraper import Scraper from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC @@ -13,7 +13,7 @@ class PosPrevias(Scraper, PlanSection): def __init__(self, driver, wait, browser: str = "firefox", debug: bool = False, home_url: str = None): Scraper.__init__(self, driver, wait, browser, debug) - link = f"{home_url}/views/public/desktop/consultarDeQueEsPrevia/consultarDeQueEsPrevia01.xhtml?cid=1" + link = f"{home_url.rstrip('/')}/views/public/desktop/consultarDeQueEsPrevia/consultarDeQueEsPrevia01.xhtml?cid=1" PlanSection.__init__(self, link) self.home_url = home_url diff --git a/scraper/pages/previas.py b/scraper/pages/previas.py index 7d62c3b..6f58328 100644 --- a/scraper/pages/previas.py +++ b/scraper/pages/previas.py @@ -425,7 +425,7 @@ def expand_all_requirements(self): while self.try_find_element( (By.XPATH, '//span[@class="ui-tree-toggler ui-icon ui-icon-plus"]'), 0.2 ): - if tries >=50: + if tries == 50: self.driver.refresh() if tries >= 90: raise Exception("Failed to expand all requirements") @@ -548,7 +548,13 @@ def _process_plan_with_retry(self, plan: str, year: str, max_retries: int = 3) - By.XPATH, '//tr[contains(@class, "ui-datatable-even") or contains(@class, "ui-datatable-odd")]' ) + if i >= len(fresh_rows): + self.logger.warning(f"Row {i} disappeared after re-render; skipping") + continue fresh_cells = fresh_rows[i].find_elements(By.TAG_NAME, "td") + if len(fresh_cells) < 3: + self.logger.warning(f"Row {i} has fewer cells than expected; skipping") + continue ver_mas_link = fresh_cells[2].find_element(By.TAG_NAME, "a") # Click Ver Más diff --git a/scraper/pages/vigentes.py b/scraper/pages/vigentes.py index fdb50ca..0e1c4ef 100644 --- a/scraper/pages/vigentes.py +++ b/scraper/pages/vigentes.py @@ -107,15 +107,17 @@ def _process_plan_with_retry(self, plan: str, year: str, max_retries: int = 3) - self.logger.info("Sacando instancias de dictado con período disponible") data_disponibles = {} - if not self.try_find_element((By.XPATH, '//td[text()= "No existen instancias de evaluación con período de inscripción/desistimiento habilitado."]')): + if not self.try_find_element((By.XPATH, '//td[contains(text(), "No existen instancias")]')): data_disponibles = self.process_table() self.remove_element(self.driver.find_element(By.XPATH, '//div[@id="accordDict:tabDictH_header"]')) self.remove_element(self.driver.find_element(By.XPATH, '//div[@id="accordDict:tabDictH"]')) self.logger.info("Sacando instancias de dictado con período finalizado") self.wait_for_element_to_be_clickable((By.XPATH, '//div[contains(text(), "Instancias de dictado con período finalizado")]')).click() sleep(1) - - data_finalizadas = self.process_table() + + data_finalizadas = {} + if not self.try_find_element((By.XPATH, '//td[contains(text(), "No existen instancias")]')): + data_finalizadas = self.process_table() combined_data = {**data_disponibles, **data_finalizadas} self.open_plan_section( diff --git a/scraper/scraper.py b/scraper/scraper.py index ddfdf1e..b8036a0 100644 --- a/scraper/scraper.py +++ b/scraper/scraper.py @@ -61,6 +61,7 @@ def scroll_to_element_and_click(self, element): """Scroll to element and click, waiting for modal to disappear first.""" # Always wait for the persistent modal overlay to disappear trys_number = 0 + last_error = None while trys_number < 3: trys_number += 1 try: @@ -70,9 +71,9 @@ def scroll_to_element_and_click(self, element): self.scroll_to_element(element) self.wait_for_element_to_be_clickable(element).click() return True - except: - pass - traceback.print_exc() + except Exception as e: + last_error = e + self.logger.warning(f"Failed to click element after 3 attempts: {last_error}") return False