python3Packages.spacy_models.*: add missing models and fix dependencies

pkgs/development/python-modules/spacy/models.json

@@ -41,6 +41,12 @@
     "sha256": "0bmbk6vnad3xqhg0jg8dhfhh75vyahsm16mn8ddzchhl7wm8axcc",
     "license": "cc-by-sa-40"
   },
+  {
+    "pname": "da_core_news_trf",
+    "version": "3.5.0",
+    "sha256": "0b8mxr1ajyw8ccm0khmcp4n3jcxl4syfrmiy9kzf3cp4hcrnqnxy",
+    "license": "cc-by-sa-40"
+  },
   {
     "pname": "de_core_news_lg",
     "version": "3.5.0",
@@ -131,6 +137,24 @@
     "sha256": "1py98kc6dxx5a6v6pc7hpldd6jm5s2a8vwp7l7d2jxadh947ma12",
     "license": "gpl3"
   },
+  {
+    "pname": "fi_core_news_lg",
+    "version": "3.5.0",
+    "sha256": "0j3r01a0yqgj8apfjv1wkblhqg86yp2nzxv51nf99pi2nmh81jzx",
+    "license": "cc-by-sa-40"
+  },
+  {
+    "pname": "fi_core_news_md",
+    "version": "3.5.0",
+    "sha256": "09qfzwyw6wfdmw1bgd1kfg1gdbmzal5z1r240djivxygzn6f1ixs",
+    "license": "cc-by-sa-40"
+  },
+  {
+    "pname": "fi_core_news_sm",
+    "version": "3.5.0",
+    "sha256": "1ly71cacy0gr62acvc3vl8dxh2czd6zkm7ijprisdblw17ik9yln",
+    "license": "cc-by-sa-40"
+  },
   {
     "pname": "fr_core_news_lg",
     "version": "3.5.0",
@@ -155,6 +179,24 @@
     "sha256": "0ciyilnc5gx0f1qakim57pizj1dknm8l8gd72avmrmzg3z52mgl2",
     "license": "lgpllr"
   },
+  {
+    "pname": "hr_core_news_lg",
+    "version": "3.5.0",
+    "sha256": "1fvkzfi539fmp6jy3hjcrwvdxw5k6zc3h351s887xidlw3gs1kr3",
+    "license": "cc-by-sa-40"
+  },
+  {
+    "pname": "hr_core_news_md",
+    "version": "3.5.0",
+    "sha256": "1mi6k9qjxbigrl2fa60blyyz8b54jda5hc1s96vn9rykg4rni8cr",
+    "license": "cc-by-sa-40"
+  },
+  {
+    "pname": "hr_core_news_sm",
+    "version": "3.5.0",
+    "sha256": "1s22mx7y5h135ry5l49az30l7mw7fdrz53s4a9gaxfsp9rzs474g",
+    "license": "cc-by-sa-40"
+  },
   {
     "pname": "it_core_news_lg",
     "version": "3.5.0",
@@ -173,6 +215,24 @@
     "sha256": "1fw262m7bl3g31gz0jb6fxrd385p67q82wfrsff6z9daxi3pi6ip",
     "license": "cc-by-nc-sa-30"
   },
+  {
+    "pname": "ko_core_news_lg",
+    "version": "3.5.0",
+    "sha256": "1q314wb114ynkf455cm8jd9jsx3yb6y0rrgf820ww31jlk5jzaa9",
+    "license": "cc-by-sa-40"
+  },
+  {
+    "pname": "ko_core_news_md",
+    "version": "3.5.0",
+    "sha256": "0dy7kk4bvjl944vv2m4hcvppar7clwq28y2rk40i3022jbqh2nxq",
+    "license": "cc-by-sa-40"
+  },
+  {
+    "pname": "ko_core_news_sm",
+    "version": "3.5.0",
+    "sha256": "1i5q8dpyfa2sy80hr81r6s9dqpawp36ni8slz035b0wd9sq3i73v",
+    "license": "cc-by-sa-40"
+  },
   {
     "pname": "lt_core_news_lg",
     "version": "3.5.0",
@@ -335,6 +395,30 @@
     "sha256": "1c0w85xn8lnx394qmmnv3px68w0pha7fxx0qlqa74r2mfi3sv6s7",
     "license": "cc-by-sa-40"
   },
+  {
+    "pname": "uk_core_news_lg",
+    "version": "3.5.0",
+    "sha256": "0hl9xjnxslckc6wvfgkj30r3py8q95yj7mrxdb6m5gvknlq72kp2",
+    "license": "mit"
+  },
+  {
+    "pname": "uk_core_news_md",
+    "version": "3.5.0",
+    "sha256": "05mg719ra5khm61yr7xhfcsh3apl29s3h2wkq0v87gkyqn13812p",
+    "license": "mit"
+  },
+  {
+    "pname": "uk_core_news_sm",
+    "version": "3.5.0",
+    "sha256": "1dkbmjbyhf6vsr7c4m4njgi969sfhbdnp73skl3k206dign5qgnz",
+    "license": "mit"
+  },
+  {
+    "pname": "uk_core_news_trf",
+    "version": "3.5.0",
+    "sha256": "02bhvcivalifrxd3vl118799wvg6hgykj31wwfdsgnq68lwc28fb",
+    "license": "mit"
+  },
   {
     "pname": "xx_ent_wiki_sm",
     "version": "3.5.0",

pkgs/development/python-modules/spacy/models.nix

@@ -1,8 +1,9 @@
 { lib
 , buildPythonPackage
 , fetchurl
-, jieba
-, pymorphy2
+, protobuf
+, pymorphy3
+, pymorphy3-dicts-uk
 , sentencepiece
 , spacy
 , spacy-pkuseg
@@ -15,8 +16,10 @@
 }:
 let
   buildModelPackage = { pname, version, sha256, license }:
+
     let
       lang = builtins.substring 0 2 pname;
+      requires-protobuf = pname == "fr_dep_news_trf" || pname == "uk_core_news_trf";
     in
     buildPythonPackage {
       inherit pname version;
@@ -27,16 +30,21 @@ let
       };
 
       propagatedBuildInputs = [ spacy ]
-        ++ lib.optionals (lang == "zh") [ jieba spacy-pkuseg ]
         ++ lib.optionals (lib.hasSuffix "_trf" pname) [ spacy-transformers ]
-        ++ lib.optionals (lang == "ru") [ pymorphy2 ]
+        ++ lib.optionals (lang == "ru") [ pymorphy3 ]
+        ++ lib.optionals (lang == "uk") [ pymorphy3 pymorphy3-dicts-uk ]
+        ++ lib.optionals (lang == "zh") [ spacy-pkuseg ]
         ++ lib.optionals (pname == "fr_dep_news_trf") [ sentencepiece ];
 
-      postPatch = lib.optionalString (pname == "fr_dep_news_trf") ''
+      postPatch = lib.optionalString requires-protobuf ''
         substituteInPlace meta.json \
-          --replace "sentencepiece==0.1.91" "sentencepiece>=0.1.91"
+          --replace "protobuf<3.21.0" "protobuf"
       '';
 
+      nativeBuildInputs = lib.optionals requires-protobuf [
+        protobuf
+      ];
+
       pythonImportsCheck = [ pname ];
 
       passthru.updateScript = writeScript "update-spacy-models" ''