Lutaml integration

.github/workflows/rake.yml

@@ -4,9 +4,10 @@ name: rake
 
 on:
   push:
-    branches: [ master, main ]
+    branches: [ master, main, lutaml-integration ]
     tags: [ v* ]
   pull_request:
+  workflow_dispatch:
 
 jobs:
   rake:

.rubocop.yml

@@ -7,6 +7,6 @@ require: rubocop-rails
 inherit_from:
   - https://raw.githubusercontent.com/riboseinc/oss-guides/master/ci/rubocop.yml
 AllCops:
-  TargetRubyVersion: 3.0
+  TargetRubyVersion: 3.2
 Rails:
   Enabled: true

Gemfile

@@ -3,6 +3,8 @@ source "https://rubygems.org"
 # Specify your gem's dependencies in gemspec
 gemspec
 
+
+
 gem "byebug", "~> 11.0"
 gem "equivalent-xml", "~> 0.6"
 gem "pry"
@@ -15,3 +17,4 @@ gem "simplecov"
 gem "vcr"
 gem "webmock"
 gem "debug"
+gem "fiddle"

lib/relaton/bibcollection.rb

@@ -32,8 +32,8 @@ def <<(item)
 
     # @param source [Nokogiri::XML::Element]
     def self.from_xml(source)
-      title = find_text("./relaton-collection/title", source)
-      author = find_text(
+      title = find_html("./relaton-collection/title", source)
+      author = find_html(
         "./relaton-collection/contributor[role/@type='author']/organization/"\
         "name", source
       )
@@ -61,10 +61,10 @@ def to_xml(opts = {})
                         end
 
       ret = "<relaton-collection #{collection_type}>"
-      ret += "<title>#{title}</title>" if title
+      ret += "<title>#{xml_escape(title)}</title>" if title
       if author
         ret += "<contributor><role type='author'/><organization><name>"\
-        "#{author}</name></organization></contributor>"
+        "#{xml_escape(author)}</name></organization></contributor>"
       end
       unless items.empty?
         items.each do |item|
@@ -134,5 +134,13 @@ def reduce_items
                end
       end
     end
+
+    # Escape bare & in content for XML serialization, leaving already-encoded
+    # entity references (&amp;, &#123;, &#x1f;) and inline markup (<em> etc.)
+    # untouched. This prevents invalid XML when plain-text values (e.g. from
+    # YAML) contain literal ampersands.
+    def xml_escape(str)
+      str.gsub(/&(?![a-zA-Z][a-zA-Z0-9]*;|#[0-9]+;|#x[0-9a-fA-F]+;)/, "&amp;")
+    end
   end
 end

lib/relaton/bibdata.rb

@@ -13,7 +13,7 @@ def initialize(bibitem)
     end
 
     def docidentifier
-      @bibitem.docidentifier.first&.id
+      @bibitem.docidentifier.first&.content&.to_s
     end
 
     # def doctype
@@ -50,10 +50,9 @@ def to_xml(opts = {})
     end
 
     def to_h
-      URL_TYPES.reduce(@bibitem.to_hash) do |h, t|
+      URL_TYPES.each_with_object(YAML.safe_load(@bibitem.to_yaml)) do |t, h|
         value = send t
-        h[t.to_s] = value
-        h
+        h[t.to_s] = value if value
       end
     end
 
@@ -68,19 +67,20 @@ def method_missing(meth, *args)
       if @bibitem.respond_to?(meth)
         @bibitem.send meth, *args
       elsif URL_TYPES.include? meth
-        link = @bibitem.link.detect do |l|
+        source = (@bibitem.source || []).detect do |l|
           l.type == meth.to_s || (meth == :uri && l.type.nil?)
         end
-        link&.content&.to_s
+        source&.content&.to_s
       elsif URL_TYPES.include? meth.match(/^\w+(?==)/).to_s.to_sym
         /^(?<type>\w+)/ =~ meth
-        link = @bibitem.link.detect do |l|
+        @bibitem.source ||= []
+        source = @bibitem.source.detect do |l|
           l.type == type || (type == "uri" && l.type.nil?)
         end
-        if link
-          link.content = args[0]
+        if source
+          source.content = args[0]
         else
-          @bibitem.link << RelatonBib::TypedUri.new(type: type, content: args[0])
+          @bibitem.source << Relaton::Bib::Uri.new(type: type, content: args[0])
         end
       else
         super

lib/relaton/cli.rb

@@ -1,3 +1,4 @@
+require "fileutils"
 require "thor"
 require "thor/hollaback"
 require_relative "cli/version"
@@ -19,6 +20,7 @@ class RelatonDb
       # @return [Relaton::Db]
       def db(dir)
         if dir
+          FileUtils.mkdir_p File.dirname(DBCONF)
           File.write DBCONF, dir, encoding: "UTF-8"
           @db = Relaton::Db.new dir, nil
         else
@@ -39,19 +41,14 @@ def dbpath
 
     class << self
       def version
-        require "relaton_bib"
-        require "relaton_iso_bib"
+        require "relaton/bib"
         registry = Relaton::Registry.instance
         puts "CLI => #{Relaton::Cli::VERSION}"
-        puts "relaton => #{Relaton::VERSION}"
-        puts "relaton-bib => #{RelatonBib::VERSION}"
-        puts "relaton-iso-bib => #{RelatonIsoBib::VERSION}"
+        puts "relaton => #{Gem.loaded_specs['relaton'].version}"
+        puts "relaton-bib => #{Gem.loaded_specs['relaton-bib'].version}"
         registry.processors.each_key do |k|
-          require k.to_s
-          klass = registry.send(:camel_case, k.to_s)
-          klass = "#{klass}::VERSION"
-          version = Kernel.const_get(klass)
-          puts "#{k.to_s.sub('_', '-')} => #{version}"
+          name = k.to_s.sub("_", "-")
+          puts "#{name} => #{Gem.loaded_specs[name].version}"
         end
       end
 
@@ -76,10 +73,11 @@ def relaton(dir)
       # @return [RelatonBib::BibliographicItem,
       #   RelatonIsoBib::IsoBibliongraphicItem]
       def parse_xml(doc)
+        doc.remove_namespaces! if doc.respond_to?(:remove_namespaces!)
         if (proc = Cli.processor(doc))
           proc.from_xml(doc.to_s)
         else
-          RelatonBib::XMLParser.from_xml(doc.to_s)
+          Relaton::Bib::Item.from_xml(doc.to_s) rescue nil
         end
       end
 

lib/relaton/cli/command.rb

@@ -140,7 +140,7 @@ def convert(file) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
         xml = Nokogiri::XML(File.read(file, encoding: "UTF-8"))
         item = Relaton::Cli.parse_xml xml
         result = if /yaml|yml/.match?(options[:format])
-                   item.to_hash.to_yaml
+                   item.to_yaml
                  else item.send "to_#{options[:format]}"
                  end
         ext = case options[:format]
@@ -226,16 +226,16 @@ def fetch_document(code, options) # rubocop:disable Metrics/CyclomaticComplexity
       return "No matching bibliographic entry found" unless doc
 
       serialize doc, options[:format]
-    rescue RelatonBib::RequestError => e
+    rescue Relaton::RequestError => e
       e.message
     end
 
-    # @param doc [RelatonBib::BibliographicItem]
+    # @param doc [Relaton::Bib::ItemData]
     # @param format [String]
     # @return [String]
     def serialize(doc, format)
       case format
-      when "yaml", "yml" then doc.to_hash.to_yaml
+      when "yaml", "yml" then doc.to_yaml
       when "bibtex" then doc.to_bibtex
       else doc.to_xml bibdata: true
       end

lib/relaton/cli/full_text_search.rb

@@ -1,10 +1,13 @@
-# require "forwardable"
+require "set"
 
 module Relaton
   class FullTextSeatch
-    # extend Forwardable
-
-    # def_delegators :@collections, :<<
+    # Instance variables added by lutaml-model that form back-references
+    # (cycles) when a model is serialized. Skipping them keeps recursive
+    # traversal of a bibitem's object graph finite.
+    INTERNAL_IVARS = %i[
+      @using_default @lutaml_register @lutaml_parent @lutaml_root @register_records
+    ].freeze
 
     # @return Regexp
     attr_reader :regex
@@ -19,8 +22,7 @@ def initialize(collection)
     def search(text)
       @regex = %{(.*?)(.{0,20})(#{text})(.{0,20})(.*)}
       @matches = @collection.items.reduce({}) do |m, item|
-        # m + results(col, rgx)
-        res = result item
+        res = result item, Set.new
         m[item.id] = res if res.any?
         m
       end
@@ -58,19 +60,24 @@ def print_attrs(attrs, indent)
     end
 
     # @param item [Relaton::Bibdata]
+    # @param seen [Set<Integer>] object_ids already visited on this path
     # @return [Hash]
-    def result(item)
+    def result(item, seen)
       if item.is_a? String
         message $~ if item.match regex
       elsif item.respond_to? :reduce
         item.reduce([]) do |m, i|
-          res = result i
+          res = result i, seen
           m << res if res && !res.empty?
           m
         end
       else
+        return {} unless seen.add?(item.object_id)
+
         item.instance_variables.reduce({}) do |m, var|
-          res = result item.instance_variable_get(var)
+          next m if INTERNAL_IVARS.include?(var)
+
+          res = result item.instance_variable_get(var), seen
           m[var.to_s.tr(":@", "")] = res if res && !res.empty?
           m
         end

lib/relaton/cli/relaton_file.rb

@@ -119,15 +119,15 @@ def extract_and_write_to_files # rubocop:disable Metrics/AbcSize, Metrics/Method
           if (bib = xml.at("//bibdata"))
             bib = nokogiri_document(bib.to_xml)
           elsif (rfc = xml.at("//rfc"))
-            require "relaton_ietf/bibxml_parser"
-            ietf = RelatonIetf::BibXMLParser.fetch_rfc rfc
+            require "relaton/ietf"
+            require "relaton/ietf/bibxml_parser"
+            ietf = Relaton::Ietf::BibXMLParser.parse_rfc rfc.to_xml
             bib = nokogiri_document(ietf.to_xml(bibdata: true))
           else
             next
           end
 
           bib.remove_namespaces!
-          bib.root.add_namespace(nil, "xmlns")
 
           bibdata = Relaton::Bibdata.from_xml(bib.root)
           if bibdata
@@ -147,8 +147,9 @@ def concatenate_files
         xml_files.flatten.reduce([]) do |mem, xml|
           doc = nokogiri_document(xml[:content])
           if (rfc = doc.at("/rfc"))
-            require "relaton_ietf/bibxml_parser"
-            ietf = RelatonIetf::BibXMLParser.fetch_rfc rfc
+            require "relaton/ietf"
+            require "relaton/ietf/bibxml_parser"
+            ietf = Relaton::Ietf::BibXMLParser.parse_rfc rfc.to_xml
             d = nokogiri_document ietf.to_xml(bibdata: true)
             mem << bibdata_instance(d, xml[:file])
           elsif %w[bibitem bibdata].include? doc&.root&.name
@@ -231,7 +232,6 @@ def build_bibdata_relaton(bibdata, file)
       # @return [Nokogiri::XML::Document]
       def clean_nokogiri_document(document)
         document.remove_namespaces!
-        document.root.add_namespace(nil, "xmlns")
         nokogiri_document(document.to_xml)
       end
 

lib/relaton/cli/subcommand_collection.rb

@@ -144,11 +144,13 @@ def import(file) # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
         collfile = File.join directory, options[:collection]
         coll = read_collection collfile
         xml = Nokogiri::XML File.read(file, encoding: "UTF-8")
+        xml.remove_namespaces!
         if xml.at "relaton-collection"
+          imported = import_collection(xml)
           if coll
-            coll << Relaton::Bibcollection.from_xml(xml)
+            coll << imported
           else
-            coll = Relaton::Bibcollection.from_xml(xml)
+            coll = imported
           end
         else
           coll ||= Relaton::Bibcollection.new({})
@@ -169,6 +171,22 @@ def export(file)
 
       private
 
+      # Parse a namespace-free collection XML document.
+      # Bibcollection.from_xml expects namespaced docs (via apply_namespace),
+      # so we use plain XPath on the namespace-stripped document instead.
+      def import_collection(xml)
+        title = xml.at("relaton-collection/title")&.text
+        author = xml.at_xpath(
+          "./relaton-collection/contributor[role/@type='author']"\
+          "/organization/name",
+        )&.text
+        items = xml.xpath("relaton-collection/relation").map do |rel|
+          el = rel.at("bibdata") || rel.at("bibitem")
+          Relaton::Bibdata.from_xml(el || rel)
+        end.compact
+        Relaton::Bibcollection.new(title: title, author: author, items: items)
+      end
+
       # @return [String]
       def directory
         options.fetch :dir, File.join(Dir.home, ".relaton/collections")
@@ -231,9 +249,9 @@ def output_item(item)
       # @param item [Relaton::Bibdata]
       def puts_human_readable_item(item) # rubocop:disable Metrics/AbcSize
         puts "Document identifier: #{item.docidentifier}"
-        puts "Title: #{item.title.first.title.content}"
+        puts "Title: #{item.title.first.content}"
         puts "Status: #{item.status.stage}"
-        item.date.each { |d| puts "Date #{d.type}: #{d.on || d.from}" }
+        item.date.each { |d| puts "Date #{d.type}: #{d.at || d.from}" }
       end
     end
   end

lib/relaton/cli/util.rb

@@ -1,7 +1,7 @@
 module Relaton
   module Cli
     module Util
-      extend RelatonBib::Util
+      extend Relaton::Bib::Util
       PROGNAME = "relaton-cli".freeze
     end
   end

lib/relaton/cli/version.rb

@@ -1,5 +1,5 @@
 module Relaton
   module Cli
-    VERSION = "1.20.6".freeze
+    VERSION = "2.1.2".freeze
   end
 end

lib/relaton/cli/xml_to_html_renderer.rb

@@ -90,12 +90,13 @@ def hash_to_liquid(hash)
           elsif value.is_a?(Hash) then v = value["content"]
           else v = value
           end
-        when "docid"
-          v = if value.is_a?(Array)
-                value.detect { |did| did["id"] !~ /^(http|https):\/\// } ||
-                  value.first
-              else value
-              end
+        when "docidentifier"
+          did = if value.is_a?(Array)
+                  value.detect { |d| (d["id"] || d["content"]).to_s !~ /^(http|https):\/\// } ||
+                    value.first
+                else value
+                end
+          v = did
         else v = value
         end
         [key.to_s, empty2nil(v)]