Redesign

AGENTS.md

@@ -160,6 +160,7 @@ Invocation (the plain `docker compose` command uses `docker-compose.yml`):
 - Define reusable Jinja2 components when patterns emerge, but avoid over-engineering for future reuse.
 - Views are organized by concern in `webtool/views/`. API endpoints are in `api_tool.py` and `api_standalone.py`.
 - Static assets go in `webtool/static/`; templates in `webtool/templates/`.
+- The project uses htmx `4.0.0-beta*`.
 
 ## Testing Expectations
 - Run tests with `pytest` from the repo root. Config is in `pytest.ini`.

backend/lib/processor.py

@@ -1,6 +1,7 @@
 """
 Basic post-processor worker - should be inherited by workers to post-process results
 """
+from dataclasses import dataclass, field
 import traceback
 import inspect as py_inspect
 import zipfile
@@ -30,6 +31,58 @@
 # Shared instance for the legacy default `compatibility`
 _DEFAULT_COMPATIBILITY = Compatibility(top_dataset_only=True)
 
+@dataclass
+class ProcessorDescription:
+    """
+    A processor's user-facing description: the information shown about it in
+    the web interface. A processor can declare one of these directly (as its
+    `description` attribute) or declare the individual attributes; either way
+    the result is available via `Processor.get_description()`.
+    """
+    title: str
+    description: str
+    category: str = ""
+    tags: typing.List[str] = field(default_factory=list)
+    references: typing.List[str] = field(default_factory=list)
+    info: typing.List[str] = field(default_factory=list)
+    warnings: typing.List[str] = field(default_factory=list)
+    icon: str = ""
+
+    def __post_init__(self):
+        # Normalise casing so processors don't have to: categories always start
+        # with a capital, tags are always lower-case. This keeps the two
+        # consistent however a processor happens to spell them.
+        if self.category:
+            self.category = self.category[0].upper() + self.category[1:]
+        self.tags = [tag.lower() for tag in self.tags]
+
+        # `category` is kept as the first entry of `tags` (as its lower-case
+        # tag form), so 4CAT can move to tags (which allow several per processor
+        # and can be filtered on) while `category` keeps working. Derive
+        # whichever is missing; when both are given, make sure the category
+        # leads the tag list.
+        category_tag = self.category.lower()
+        if self.category and not self.tags:
+            self.tags = [category_tag]
+        elif self.tags and not self.category:
+            self.category = self.tags[0][0].upper() + self.tags[0][1:]
+        elif self.category and self.tags:
+            self.tags = [category_tag] + [tag for tag in self.tags if tag != category_tag]
+
+
+class _DescriptionField:
+    """
+    Exposes one ProcessorDescription field as an attribute on the processor,
+    e.g. `Processor.title`. A plain `property` only runs on instance access;
+    this descriptor also runs on class access (`owner` is the class in both
+    cases), so `Processor.title` and `self.title` both return the value from
+    the processor's ProcessorDescription.
+    """
+    def __init__(self, name):
+        self.name = name
+
+    def __get__(self, obj, owner):
+        return getattr(owner._processor_description, self.name)
 
 class BasicProcessor(FourcatModule, BasicWorker, metaclass=abc.ABCMeta):
     """
@@ -81,11 +134,31 @@ def is_compatible_with(cls, module=None, config=None):
     #: The file that is being processed
     source_file = None
 
-    #: Processor description, which will be displayed in the web interface
-    description = "No description available"
-
-    #: Category identifier, used to group processors in the web interface
-    category = "Other"
+    #: The processor's user-facing description (title, category, description
+    #: text and references) as a single object. A processor may set this
+    #: directly, or set the individual attributes below; both are normalised
+    #: into `_processor_description` when the class is defined.
+    _processor_description = ProcessorDescription(
+        title="", 
+        category="Other", 
+        description="No description available", 
+        references=[],
+        info=[],
+        warnings=[],
+        icon=""
+        )
+
+    #: Title, category, description text and references, read from the
+    #: processor's ProcessorDescription. Defined as descriptors so that both
+    #: `Processor.title` and `self.title` resolve to the stored value.
+    title = _DescriptionField("title")
+    category = _DescriptionField("category")
+    description = _DescriptionField("description")
+    references = _DescriptionField("references")
+    info = _DescriptionField("info")
+    warnings = _DescriptionField("warnings")
+    tags = _DescriptionField("tags")
+    icon = _DescriptionField("icon")
 
     #: Extension of the file created by the processor
     extension = "csv"
@@ -991,6 +1064,18 @@ def _validate_map_item_post_run(self):
             except Exception:
                 pass
 
+    @classmethod
+    def get_repo_link(cls, config):
+        """
+        Get a link to the processor's source code repository
+
+        :param ConfigManager config:  Configuration reader
+        :return str:  URL to the processor's source code repository
+        """
+        repo_url = config.get("4cat.github_url")
+        path = cls.filepath.replace("\\", "/").lstrip("/")
+        return f"{repo_url.rstrip('/')}/blob/master/{path}"
+
     @classmethod
     def is_compatible_with(cls, module=None, config=None):
         """
@@ -1134,6 +1219,58 @@ def exclude_followup_processors(cls, processor_type=None):
             return True
         return False
 
+    def __init_subclass__(cls, **kwargs):
+        """
+        Normalise a processor's description when its class is defined.
+
+        A processor may declare its description either as a ProcessorDescription
+        object (assigned to `description`) or as the individual attributes
+        (title, category, description, references, info, warnings, icon, tags).
+        Either way it is folded into a single `_processor_description` object
+        here, and the raw attributes are removed so the descriptors on
+        BasicProcessor provide access to them.
+        """
+        super().__init_subclass__(**kwargs)
+
+        # the description inherited from the nearest ancestor (cls has none yet)
+        inherited = getattr(cls, "_processor_description", None)
+
+        # read the raw class-body value, bypassing the descriptor
+        declared = cls.__dict__.get("description")
+        if isinstance(declared, ProcessorDescription):
+            description = declared
+        else:
+            # build from the flat attributes, falling back to inherited values
+            # so a legacy subclass keeps anything an ancestor set
+            description = ProcessorDescription(
+                title=cls.__dict__.get("title", inherited.title),
+                category=cls.__dict__.get("category", inherited.category),
+                description=cls.__dict__.get("description", inherited.description),
+                references=list(cls.__dict__.get("references", inherited.references)),
+                info=list(cls.__dict__.get("info", inherited.info)),
+                warnings=list(cls.__dict__.get("warnings", inherited.warnings)),
+                # tags default to this class's category (see __post_init__), so
+                # don't inherit them — re-derive from the resolved category
+                tags=list(cls.__dict__.get("tags", [])),
+                icon=cls.__dict__.get("icon", inherited.icon),   
+            )
+
+        # remove raw attributes so the inherited descriptors govern access
+        for name in ("title", "category", "description", "references", "info", "warnings", "tags", "icon"):
+            if name in cls.__dict__:
+                delattr(cls, name)
+
+        cls._processor_description = description
+
+    @classmethod
+    def get_description(cls):
+        """
+        Get the processor's user-facing description
+
+        :return ProcessorDescription:  Description of this processor
+        """
+        return cls._processor_description
+
     @abc.abstractmethod
     def process(self):
         """
@@ -1162,4 +1299,4 @@ def is_preset():
 
         :return:  False
         """
-        return False
+        return False

backend/lib/search.py

@@ -13,6 +13,7 @@
 from backend.lib.processor import BasicProcessor
 from common.lib.helpers import strip_tags, dict_search_and_update, remove_nuls, HashCache, format_import_item
 from common.lib.exceptions import WorkerInterruptedException, ProcessorInterruptedException, MapItemException
+from common.lib.outputs import Datasource
 
 
 class Search(BasicProcessor, ABC):
@@ -32,6 +33,15 @@ class Search(BasicProcessor, ABC):
 	#: backwards-compatibility reasons. For example, `instagram-search`.
 	type = "abstract-search"
 
+	#: Default output shape: a collected, top-level dataset whose extension is this
+	#: worker's own (ndjson for most, csv/zip for some). A data source that produces
+	#: media (an uploaded archive) or whose shape is only known at run time overrides
+	#: this with a MediaArchive or a per-worker Output.
+	output = Datasource()
+
+	# generic icon
+	icon = "comments"
+
 	#: Amount of workers of this type that can run in parallel. Be careful with
 	#: this, because values higher than 1 will mean that e.g. API rate limits
 	#: are easily violated.

backend/workers/datasource_metrics.py

@@ -10,7 +10,7 @@
 """
 import os
 
 from datetime import datetime, time, timezone
 
 from backend.lib.worker import BasicWorker
 
@@ -38,7 +38,6 @@
 
     def work(self):
         self.general_stats()
-        self.data_stats()
 
     @staticmethod
     def folder_size(path='.'):
@@ -77,137 +76,4 @@
                 "datasource": "4cat",
                 "board": "",
                 "date": "now"
-            }, constraints=["metric", "datasource", "board", "date"])
-
-    def data_stats(self):
-        """
-        Go through all local datasources, and update the posts per day
-        if they haven't been calculated yet. These data can then be used
-        to calculate e.g. posts per month.
-        :return:
-        """
-
-        # Get a list of all database tables
-        all_tables = [row["tablename"] for row in self.db.fetchall(
-            "SELECT tablename FROM pg_catalog.pg_tables WHERE schemaname != 'pg_catalog' AND schemaname != 'information_schema';")]
-
-        # Check if the metrics table is already present
-        metrics_exists = True if "metrics" in all_tables else False
-
-        # If not, make it.
-        if not metrics_exists:
-            self.db.execute("""
-				CREATE TABLE IF NOT EXISTS metrics (
-				  metric             text,
-				  datasource         text,
-				  board              text,
-				  date               text,
-				  count              integer
-				);
-
-			""")
-
-        added_datasources = [row["datasource"] for row in self.db.fetchall("SELECT DISTINCT(datasource) FROM metrics")]
-        enabled_datasources = self.config.get("datasources.enabled", {})
-
-        for datasource_id in self.modules.datasources:
-            if datasource_id not in enabled_datasources:
-                continue
-
-            datasource = self.modules.workers.get(datasource_id + "-search")
-            if not datasource:
-                continue
-
-            # Database IDs may be different from the Datasource ID (e.g. the datasource "4chan" became "fourchan" but the database ID remained "4chan")
-            database_db_id = datasource.prefix if hasattr(datasource, "prefix") else datasource_id
-
-            is_local = True if hasattr(datasource, "is_local") and datasource.is_local else False
-            is_static = True if hasattr(datasource, "is_static") and datasource.is_static else False
-
-            # Only update local datasources
-            if is_local:
-
-                # Some translating..
-                settings_id = datasource_id
-                if datasource_id == "4chan":
-                    settings_id = "fourchan"
-                elif datasource_id == "8chan":
-                    settings_id = "eightchan"
-
-                boards = [b for b in self.config.get(settings_id + "-search.boards", [])]
-
-                # If a datasource is static (so not updated) and it
-                # is already present in the metrics table, we don't
-                # need to update its metrics anymore.
-                if is_static and datasource_id in added_datasources:
-                    continue
-                else:
-
-                    # -------------------------
-                    #   Posts per day metric
-                    # -------------------------
-
-                    # Get the name of the posts table for this datasource
-                    posts_table = datasource_id if "posts_" + database_db_id not in all_tables else "posts_" + database_db_id
-
-                    # Count and update for every board individually
-                    for board in boards:
-
-                        if not board:
-                            board_sql = " board = '' OR board = NULL"
-                        else:
-                            board_sql = " board='" + board + "'"
-
-                        # Midnight of this day in UTC epoch timestamp
-                        midnight = int(
-                            datetime.combine(datetime.today(), time.min).replace(tzinfo=timezone.utc).timestamp())
-
-                        # We only count passed days
-                        time_sql = "timestamp < " + str(midnight)
-
-                        # If the datasource is dynamic, we also only update days
-                        # that haven't been added yet - these are heavy queries.
-                        if not is_static:
-                            days_added = self.db.fetchall(
-                                "SELECT date FROM metrics WHERE datasource = '%s' AND board = '%s' AND metric = 'posts_per_day';" % (
-                                database_db_id, board))
-
-                            if days_added:
-
-                                last_day_added = max([row["date"] for row in days_added])
-                                last_day_added = datetime.strptime(last_day_added, '%Y-%m-%d').replace(
-                                    tzinfo=timezone.utc)
-
-                                # If the last day added is today, there's no need to update yet
-                                if last_day_added.date() == datetime.today().replace(tzinfo=timezone.utc).date():
-                                    self.log.info(
-                                        "No new posts per day to count for %s%s" % (datasource_id, "/" + board))
-                                    continue
-
-                                # Change to UTC epoch timestamp for postgres query
-                                after_timestamp = int(last_day_added.timestamp())
-
-                                time_sql += " AND timestamp > " + str(after_timestamp) + " "
-
-                        self.log.info(
-                            "Calculating metric posts_per_day for datasource %s%s" % (datasource_id, "/" + board))
-
-                        # Get those counts
-                        query = """
-							SELECT 'posts_per_day' AS metric, '%s' AS datasource, board, to_char(to_timestamp(timestamp), 'YYYY-MM-DD') AS date, count(*)COUNT
-							FROM %s
-							WHERE %s AND %s
-							GROUP BY metric, datasource, board, date;
-							""" % (database_db_id, posts_table, board_sql, time_sql)
-                        # Add to metrics table
-                        rows = [dict(row) for row in self.db.fetchall(query)]
-
-                        if rows:
-                            for row in rows:
-                                self.db.upsert("metrics", row, constraints=["metric", "datasource", "board", "date"])
-
-                # -------------------------------
-                #   no other metrics added yet
-                # -------------------------------
-
-        self.job.finish()
+            }, constraints=["metric", "datasource", "board", "date"])