Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 14 additions & 8 deletions addons/metadata/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,19 @@
class FileMetadataMigrator:
"""Migrate file metadata across FileMetadata, Registration, and DraftRegistration.

Runs inside data migrations: models are resolved through the historical
``apps`` registry and saved without custom save()/signals, so later schema
changes and search indexing cannot break the migration.

transform_item: fn(item_data: dict) -> bool
Mutates item_data in-place. Returns True if changed.
transform_filemetadata_entry: fn(entry: dict) -> bool
Mutates a filemetadata entry (with 'metadata' key) in-place. Returns True if changed.
Used for Registration/DraftRegistration where filemetadata is nested.
"""

def __init__(self, schema_name, transform_item, transform_filemetadata_entry=None):
def __init__(self, apps, schema_name, transform_item, transform_filemetadata_entry=None):
self.apps = apps
self.schema_name = schema_name
self.transform_item = transform_item
self.transform_filemetadata_entry = transform_filemetadata_entry or self._default_transform_entry
Expand All @@ -31,6 +36,7 @@ def _default_transform_entry(entry):
return False

def _get_schema_ids(self):
RegistrationSchema = self.apps.get_model('osf', 'RegistrationSchema')
schemas = list(RegistrationSchema.objects.filter(name=self.schema_name))
if not schemas:
logger.warning(f'Skipped: No schema found for "{self.schema_name}"')
Expand All @@ -46,12 +52,12 @@ def run(self):
self._migrate_draft_registration(schema_ids)

def _migrate_filemetadata(self, schema_ids):
from addons.metadata.models import FileMetadata
FileMetadata = self.apps.get_model('addons_metadata', 'FileMetadata')
for fm in FileMetadata.objects.filter(metadata__isnull=False, deleted__isnull=True):
try:
metadata = json.loads(fm.metadata)
except json.JSONDecodeError:
logger.warning(f'Skipped: bad JSON for FileMetadata {fm._id}', exc_info=True)
logger.warning(f'Skipped: bad JSON for FileMetadata {fm.id}', exc_info=True)
continue
dirty = False
for item in metadata.get('items', []):
Expand All @@ -62,10 +68,10 @@ def _migrate_filemetadata(self, schema_ids):
if dirty:
fm.metadata = json.dumps(metadata, ensure_ascii=False)
fm.save()
logger.info(f'Migrated FileMetadata {fm._id} path="{fm.path}"')
logger.info(f'Migrated FileMetadata {fm.id} path="{fm.path}"')

def _migrate_registration(self, schema_ids):
from osf.models import Registration
Registration = self.apps.get_model('osf', 'Registration')
registrations = Registration.objects.filter(
registered_meta__isnull=False,
registered_schema__name=self.schema_name,
Expand All @@ -76,7 +82,7 @@ def _migrate_registration(self, schema_ids):
try:
filemetadatas = json.loads(meta_value.get('grdm-files', {}).get('value', '[]'))
except json.JSONDecodeError:
logger.warning(f'Skipped: bad JSON for Registration {reg._id}', exc_info=True)
logger.warning(f'Skipped: bad JSON for Registration {reg.id}', exc_info=True)
continue
entry_dirty = False
for entry in filemetadatas:
Expand All @@ -88,10 +94,10 @@ def _migrate_registration(self, schema_ids):
dirty = True
if dirty:
reg.save()
logger.info(f'Migrated Registration {reg._id}')
logger.info(f'Migrated Registration {reg.id}')

def _migrate_draft_registration(self, schema_ids):
from osf.models import DraftRegistration
DraftRegistration = self.apps.get_model('osf', 'DraftRegistration')
drafts = DraftRegistration.objects.filter(
registration_metadata__isnull=False,
registration_schema__name=self.schema_name,
Expand Down
3 changes: 2 additions & 1 deletion osf/migrations/0267_split_name_fields.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,14 @@
TARGET_SCHEMA_NAME = '公的資金による研究データのメタデータ登録'


def migrate_name_fields(*args):
def migrate_name_fields(apps, schema_editor):
from addons.metadata.utils import (
FileMetadataMigrator,
transform_name_fields_item,
transform_name_fields_entry,
)
migrator = FileMetadataMigrator(
apps,
TARGET_SCHEMA_NAME,
transform_name_fields_item,
transform_name_fields_entry,
Expand Down
3 changes: 2 additions & 1 deletion osf/migrations/0270_remove_metadata_access_rights.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@ def _transform_entry(entry):
return False


def remove_metadata_access_rights(*args):
def remove_metadata_access_rights(apps, schema_editor):
from addons.metadata.utils import FileMetadataMigrator
migrator = FileMetadataMigrator(
apps,
TARGET_SCHEMA_NAME,
_transform_item,
_transform_entry,
Expand Down
Loading