mirror of
https://github.com/NixOS/nixpkgs.git
synced 2025-01-31 01:04:25 +00:00
treewide: remove some unreferenced patches
Signed-off-by: Paul Meyer <49727155+katexochen@users.noreply.github.com>
This commit is contained in:
Paul Meyer
parent
d6b4401cdf
commit
4516a96db4
@ -1,12 +0,0 @@
|
||||
diff --git a/nyxt.asd b/nyxt.asd
|
||||
index ea2630ce..fdf837e4 100644
|
||||
--- a/nyxt.asd
|
||||
+++ b/nyxt.asd
|
||||
@@ -480,7 +480,6 @@ The renderer is configured from NYXT_RENDERER or `*nyxt-renderer*'."))
|
||||
:defsystem-depends-on ("nasdf")
|
||||
:class :nasdf-system
|
||||
:depends-on (nyxt/gi-gtk)
|
||||
- :build-operation "program-op"
|
||||
:build-pathname "nyxt"
|
||||
:entry-point "nyxt:entry-point")
|
||||
|
||||
@ -1,360 +0,0 @@
|
||||
Based on upstream 66b7fb630903fdcf3e83b6b6d56d82e904264a20, adjusted to
|
||||
apply to 1.15.0 & avoid implicit inclusion of changes from other
|
||||
intermediate commits
|
||||
|
||||
diff --git a/onnx/checker.cc b/onnx/checker.cc
|
||||
index fac56f56..c9fda9b2 100644
|
||||
--- a/onnx/checker.cc
|
||||
+++ b/onnx/checker.cc
|
||||
@@ -13,7 +13,6 @@
|
||||
#include <vector>
|
||||
|
||||
#include "onnx/common/file_utils.h"
|
||||
-#include "onnx/common/path.h"
|
||||
#include "onnx/defs/schema.h"
|
||||
#include "onnx/defs/tensor_proto_util.h"
|
||||
#include "onnx/proto_utils.h"
|
||||
@@ -135,85 +134,7 @@ void check_tensor(const TensorProto& tensor, const CheckerContext& ctx) {
|
||||
for (const StringStringEntryProto& entry : tensor.external_data()) {
|
||||
if (entry.has_key() && entry.has_value() && entry.key() == "location") {
|
||||
has_location = true;
|
||||
-#ifdef _WIN32
|
||||
- auto file_path = std::filesystem::path(utf8str_to_wstring(entry.value()));
|
||||
- if (file_path.is_absolute()) {
|
||||
- fail_check(
|
||||
- "Location of external TensorProto ( tensor name: ",
|
||||
- tensor.name(),
|
||||
- ") should be a relative path, but it is an absolute path: ",
|
||||
- entry.value());
|
||||
- }
|
||||
- auto relative_path = file_path.lexically_normal().make_preferred().wstring();
|
||||
- // Check that normalized relative path contains ".." on Windows.
|
||||
- if (relative_path.find(L"..", 0) != std::string::npos) {
|
||||
- fail_check(
|
||||
- "Data of TensorProto ( tensor name: ",
|
||||
- tensor.name(),
|
||||
- ") should be file inside the ",
|
||||
- ctx.get_model_dir(),
|
||||
- ", but the '",
|
||||
- entry.value(),
|
||||
- "' points outside the directory");
|
||||
- }
|
||||
- std::wstring data_path = path_join(utf8str_to_wstring(ctx.get_model_dir()), relative_path);
|
||||
- struct _stat64 buff;
|
||||
- if (_wstat64(data_path.c_str(), &buff) != 0) {
|
||||
- fail_check(
|
||||
- "Data of TensorProto ( tensor name: ",
|
||||
- tensor.name(),
|
||||
- ") should be stored in ",
|
||||
- entry.value(),
|
||||
- ", but it doesn't exist or is not accessible.");
|
||||
- }
|
||||
-#else // POSIX
|
||||
- if (entry.value().empty()) {
|
||||
- fail_check("Location of external TensorProto ( tensor name: ", tensor.name(), ") should not be empty.");
|
||||
- } else if (entry.value()[0] == '/') {
|
||||
- fail_check(
|
||||
- "Location of external TensorProto ( tensor name: ",
|
||||
- tensor.name(),
|
||||
- ") should be a relative path, but it is an absolute path: ",
|
||||
- entry.value());
|
||||
- }
|
||||
- std::string relative_path = clean_relative_path(entry.value());
|
||||
- // Check that normalized relative path contains ".." on POSIX
|
||||
- if (relative_path.find("..", 0) != std::string::npos) {
|
||||
- fail_check(
|
||||
- "Data of TensorProto ( tensor name: ",
|
||||
- tensor.name(),
|
||||
- ") should be file inside the ",
|
||||
- ctx.get_model_dir(),
|
||||
- ", but the '",
|
||||
- entry.value(),
|
||||
- "' points outside the directory");
|
||||
- }
|
||||
- std::string data_path = path_join(ctx.get_model_dir(), relative_path);
|
||||
- // use stat64 to check whether the file exists
|
||||
-#if defined(__APPLE__) || defined(__wasm__) || !defined(__GLIBC__)
|
||||
- struct stat buffer; // APPLE, wasm and non-glic stdlibs do not have stat64
|
||||
- if (stat((data_path).c_str(), &buffer) != 0) {
|
||||
-#else
|
||||
- struct stat64 buffer; // All POSIX under glibc except APPLE and wasm have stat64
|
||||
- if (stat64((data_path).c_str(), &buffer) != 0) {
|
||||
-#endif
|
||||
- fail_check(
|
||||
- "Data of TensorProto ( tensor name: ",
|
||||
- tensor.name(),
|
||||
- ") should be stored in ",
|
||||
- data_path,
|
||||
- ", but it doesn't exist or is not accessible.");
|
||||
- }
|
||||
- // Do not allow symlinks or directories.
|
||||
- if (!S_ISREG(buffer.st_mode)) {
|
||||
- fail_check(
|
||||
- "Data of TensorProto ( tensor name: ",
|
||||
- tensor.name(),
|
||||
- ") should be stored in ",
|
||||
- data_path,
|
||||
- ", but it is not regular file.");
|
||||
- }
|
||||
-#endif
|
||||
+ resolve_external_data_location(ctx.get_model_dir(), entry.value(), tensor.name());
|
||||
}
|
||||
}
|
||||
if (!has_location) {
|
||||
@@ -1054,6 +975,93 @@ void check_model(const ModelProto& model, bool full_check, bool skip_opset_compa
|
||||
}
|
||||
}
|
||||
|
||||
+std::string resolve_external_data_location(
|
||||
+ const std::string& base_dir,
|
||||
+ const std::string& location,
|
||||
+ const std::string& tensor_name) {
|
||||
+#ifdef _WIN32
|
||||
+ auto file_path = std::filesystem::path(utf8str_to_wstring(location));
|
||||
+ if (file_path.is_absolute()) {
|
||||
+ fail_check(
|
||||
+ "Location of external TensorProto ( tensor name: ",
|
||||
+ tensor_name,
|
||||
+ ") should be a relative path, but it is an absolute path: ",
|
||||
+ location);
|
||||
+ }
|
||||
+ auto relative_path = file_path.lexically_normal().make_preferred().wstring();
|
||||
+ // Check that normalized relative path contains ".." on Windows.
|
||||
+ if (relative_path.find(L"..", 0) != std::string::npos) {
|
||||
+ fail_check(
|
||||
+ "Data of TensorProto ( tensor name: ",
|
||||
+ tensor_name,
|
||||
+ ") should be file inside the ",
|
||||
+ base_dir,
|
||||
+ ", but the '",
|
||||
+ location,
|
||||
+ "' points outside the directory");
|
||||
+ }
|
||||
+ std::wstring data_path = path_join(utf8str_to_wstring(base_dir), relative_path);
|
||||
+ struct _stat64 buff;
|
||||
+ if (_wstat64(data_path.c_str(), &buff) != 0) {
|
||||
+ fail_check(
|
||||
+ "Data of TensorProto ( tensor name: ",
|
||||
+ tensor_name,
|
||||
+ ") should be stored in ",
|
||||
+ location,
|
||||
+ ", but it doesn't exist or is not accessible.");
|
||||
+ }
|
||||
+ return wstring_to_utf8str(data_path);
|
||||
+#else // POSIX
|
||||
+ if (location.empty()) {
|
||||
+ fail_check("Location of external TensorProto ( tensor name: ", tensor_name, ") should not be empty.");
|
||||
+ } else if (location[0] == '/') {
|
||||
+ fail_check(
|
||||
+ "Location of external TensorProto ( tensor name: ",
|
||||
+ tensor_name,
|
||||
+ ") should be a relative path, but it is an absolute path: ",
|
||||
+ location);
|
||||
+ }
|
||||
+ std::string relative_path = clean_relative_path(location);
|
||||
+ // Check that normalized relative path contains ".." on POSIX
|
||||
+ if (relative_path.find("..", 0) != std::string::npos) {
|
||||
+ fail_check(
|
||||
+ "Data of TensorProto ( tensor name: ",
|
||||
+ tensor_name,
|
||||
+ ") should be file inside the ",
|
||||
+ base_dir,
|
||||
+ ", but the '",
|
||||
+ location,
|
||||
+ "' points outside the directory");
|
||||
+ }
|
||||
+ std::string data_path = path_join(base_dir, relative_path);
|
||||
+ // use stat64 to check whether the file exists
|
||||
+#if defined(__APPLE__) || defined(__wasm__) || !defined(__GLIBC__)
|
||||
+ struct stat buffer; // APPLE, wasm and non-glic stdlibs do not have stat64
|
||||
+ if (stat((data_path).c_str(), &buffer) != 0) {
|
||||
+#else
|
||||
+ struct stat64 buffer; // All POSIX under glibc except APPLE and wasm have stat64
|
||||
+ if (stat64((data_path).c_str(), &buffer) != 0) {
|
||||
+#endif
|
||||
+ fail_check(
|
||||
+ "Data of TensorProto ( tensor name: ",
|
||||
+ tensor_name,
|
||||
+ ") should be stored in ",
|
||||
+ data_path,
|
||||
+ ", but it doesn't exist or is not accessible.");
|
||||
+ }
|
||||
+ // Do not allow symlinks or directories.
|
||||
+ if (!S_ISREG(buffer.st_mode)) {
|
||||
+ fail_check(
|
||||
+ "Data of TensorProto ( tensor name: ",
|
||||
+ tensor_name,
|
||||
+ ") should be stored in ",
|
||||
+ data_path,
|
||||
+ ", but it is not regular file.");
|
||||
+ }
|
||||
+ return data_path;
|
||||
+#endif
|
||||
+}
|
||||
+
|
||||
std::set<std::string> experimental_ops = {
|
||||
"ATen",
|
||||
"Affine",
|
||||
diff --git a/onnx/checker.h b/onnx/checker.h
|
||||
index 6796acab..83012213 100644
|
||||
--- a/onnx/checker.h
|
||||
+++ b/onnx/checker.h
|
||||
@@ -160,7 +160,10 @@ void check_model_local_functions(
|
||||
|
||||
void check_model(const ModelProto& model, bool full_check = false, bool skip_opset_compatibility_check = false);
|
||||
void check_model(const std::string& model_path, bool full_check = false, bool skip_opset_compatibility_check = false);
|
||||
-
|
||||
+std::string resolve_external_data_location(
|
||||
+ const std::string& base_dir,
|
||||
+ const std::string& location,
|
||||
+ const std::string& tensor_name);
|
||||
bool check_is_experimental_op(const NodeProto& node);
|
||||
|
||||
} // namespace checker
|
||||
diff --git a/onnx/common/path.h b/onnx/common/path.h
|
||||
index 6eaf5e67..09212747 100644
|
||||
--- a/onnx/common/path.h
|
||||
+++ b/onnx/common/path.h
|
||||
@@ -31,11 +31,22 @@ inline std::wstring utf8str_to_wstring(const std::string& utf8str) {
|
||||
if (utf8str.size() > INT_MAX) {
|
||||
fail_check("utf8str_to_wstring: string is too long for converting to wstring.");
|
||||
}
|
||||
- int size_required = MultiByteToWideChar(CP_UTF8, 0, utf8str.c_str(), (int)utf8str.size(), NULL, 0);
|
||||
+ int size_required = MultiByteToWideChar(CP_UTF8, 0, utf8str.c_str(), static_cast<int>(utf8str.size()), NULL, 0);
|
||||
std::wstring ws_str(size_required, 0);
|
||||
- MultiByteToWideChar(CP_UTF8, 0, utf8str.c_str(), (int)utf8str.size(), &ws_str[0], size_required);
|
||||
+ MultiByteToWideChar(CP_UTF8, 0, utf8str.c_str(), static_cast<int>(utf8str.size()), &ws_str[0], size_required);
|
||||
return ws_str;
|
||||
}
|
||||
+inline std::string wstring_to_utf8str(const std::wstring& ws_str) {
|
||||
+ if (ws_str.size() > INT_MAX) {
|
||||
+ fail_check("wstring_to_utf8str: string is too long for converting to UTF-8.");
|
||||
+ }
|
||||
+ int size_required =
|
||||
+ WideCharToMultiByte(CP_UTF8, 0, ws_str.c_str(), static_cast<int>(ws_str.size()), NULL, 0, NULL, NULL);
|
||||
+ std::string utf8str(size_required, 0);
|
||||
+ WideCharToMultiByte(
|
||||
+ CP_UTF8, 0, ws_str.c_str(), static_cast<int>(ws_str.size()), &utf8str[0], size_required, NULL, NULL);
|
||||
+ return utf8str;
|
||||
+}
|
||||
|
||||
#else
|
||||
std::string path_join(const std::string& origin, const std::string& append);
|
||||
diff --git a/onnx/cpp2py_export.cc b/onnx/cpp2py_export.cc
|
||||
index bc2594db..83cea68f 100644
|
||||
--- a/onnx/cpp2py_export.cc
|
||||
+++ b/onnx/cpp2py_export.cc
|
||||
@@ -545,6 +545,8 @@ PYBIND11_MODULE(onnx_cpp2py_export, onnx_cpp2py_export) {
|
||||
"full_check"_a = false,
|
||||
"skip_opset_compatibility_check"_a = false);
|
||||
|
||||
+ checker.def("_resolve_external_data_location", &checker::resolve_external_data_location);
|
||||
+
|
||||
// Submodule `version_converter`
|
||||
auto version_converter = onnx_cpp2py_export.def_submodule("version_converter");
|
||||
version_converter.doc() = "VersionConverter submodule";
|
||||
diff --git a/onnx/external_data_helper.py b/onnx/external_data_helper.py
|
||||
index bbc2717f..05c486c6 100644
|
||||
--- a/onnx/external_data_helper.py
|
||||
+++ b/onnx/external_data_helper.py
|
||||
@@ -8,6 +8,7 @@ import uuid
|
||||
from itertools import chain
|
||||
from typing import Callable, Iterable, Optional
|
||||
|
||||
+import onnx.onnx_cpp2py_export.checker as c_checker
|
||||
from onnx.onnx_pb import AttributeProto, GraphProto, ModelProto, TensorProto
|
||||
|
||||
|
||||
@@ -39,9 +40,9 @@ def load_external_data_for_tensor(tensor: TensorProto, base_dir: str) -> None:
|
||||
base_dir: directory that contains the external data.
|
||||
"""
|
||||
info = ExternalDataInfo(tensor)
|
||||
- file_location = _sanitize_path(info.location)
|
||||
- external_data_file_path = os.path.join(base_dir, file_location)
|
||||
-
|
||||
+ external_data_file_path = c_checker._resolve_external_data_location( # type: ignore[attr-defined]
|
||||
+ base_dir, info.location, tensor.name
|
||||
+ )
|
||||
with open(external_data_file_path, "rb") as data_file:
|
||||
if info.offset:
|
||||
data_file.seek(info.offset)
|
||||
@@ -259,14 +260,6 @@ def _get_attribute_tensors(onnx_model_proto: ModelProto) -> Iterable[TensorProto
|
||||
yield from _get_attribute_tensors_from_graph(onnx_model_proto.graph)
|
||||
|
||||
|
||||
-def _sanitize_path(path: str) -> str:
|
||||
- """Remove path components which would allow traversing up a directory tree from a base path.
|
||||
-
|
||||
- Note: This method is currently very basic and should be expanded.
|
||||
- """
|
||||
- return path.lstrip("/.")
|
||||
-
|
||||
-
|
||||
def _is_valid_filename(filename: str) -> bool:
|
||||
"""Utility to check whether the provided filename is valid."""
|
||||
exp = re.compile('^[^<>:;,?"*|/]+$')
|
||||
diff --git a/onnx/test/test_external_data.py b/onnx/test/test_external_data.py
|
||||
index 63f6b4ef..bb14d279 100644
|
||||
--- a/onnx/test/test_external_data.py
|
||||
+++ b/onnx/test/test_external_data.py
|
||||
@@ -3,6 +3,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
from __future__ import annotations
|
||||
|
||||
+import itertools
|
||||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
@@ -204,6 +205,52 @@ class TestLoadExternalDataSingleFile(TestLoadExternalDataBase):
|
||||
attribute_tensor = new_model.graph.node[0].attribute[0].t
|
||||
np.testing.assert_allclose(to_array(attribute_tensor), self.attribute_value)
|
||||
|
||||
+ @parameterized.parameterized.expand(itertools.product((True, False), (True, False)))
|
||||
+ def test_save_external_invalid_single_file_data_and_check(
|
||||
+ self, use_absolute_path: bool, use_model_path: bool
|
||||
+ ) -> None:
|
||||
+ model = onnx.load_model(self.model_filename, self.serialization_format)
|
||||
+
|
||||
+ model_dir = os.path.join(self.temp_dir, "save_copy")
|
||||
+ os.mkdir(model_dir)
|
||||
+
|
||||
+ traversal_external_data_dir = os.path.join(
|
||||
+ self.temp_dir, "invlid_external_data"
|
||||
+ )
|
||||
+ os.mkdir(traversal_external_data_dir)
|
||||
+
|
||||
+ if use_absolute_path:
|
||||
+ traversal_external_data_location = os.path.join(
|
||||
+ traversal_external_data_dir, "tensors.bin"
|
||||
+ )
|
||||
+ else:
|
||||
+ traversal_external_data_location = "../invlid_external_data/tensors.bin"
|
||||
+
|
||||
+ external_data_dir = os.path.join(self.temp_dir, "external_data")
|
||||
+ os.mkdir(external_data_dir)
|
||||
+ new_model_filepath = os.path.join(model_dir, "model.onnx")
|
||||
+
|
||||
+ def convert_model_to_external_data_no_check(model: ModelProto, location: str):
|
||||
+ for tensor in model.graph.initializer:
|
||||
+ if tensor.HasField("raw_data"):
|
||||
+ set_external_data(tensor, location)
|
||||
+
|
||||
+ convert_model_to_external_data_no_check(
|
||||
+ model,
|
||||
+ location=traversal_external_data_location,
|
||||
+ )
|
||||
+
|
||||
+ onnx.save_model(model, new_model_filepath, self.serialization_format)
|
||||
+ if use_model_path:
|
||||
+ with self.assertRaises(onnx.checker.ValidationError):
|
||||
+ _ = onnx.load_model(new_model_filepath, self.serialization_format)
|
||||
+ else:
|
||||
+ onnx_model = onnx.load_model(
|
||||
+ new_model_filepath, self.serialization_format, load_external_data=False
|
||||
+ )
|
||||
+ with self.assertRaises(onnx.checker.ValidationError):
|
||||
+ load_external_data_for_model(onnx_model, external_data_dir)
|
||||
+
|
||||
|
||||
@parameterized.parameterized_class(
|
||||
[
|
||||
@ -1,25 +0,0 @@
|
||||
From 02266a00ce0eb6a089e7efe07816da1aa5152fc9 Mon Sep 17 00:00:00 2001
|
||||
From: Maksim Terpilovskii <maximtrp@gmail.com>
|
||||
Date: Sun, 31 Jul 2022 12:25:14 +0300
|
||||
Subject: [PATCH] increased abs tolerance for wilcoxon test
|
||||
|
||||
---
|
||||
tests/test_posthocs.py | 2 +-
|
||||
1 file changed, 1 insertion(+), 1 deletion(-)
|
||||
|
||||
diff --git a/tests/test_posthocs.py b/tests/test_posthocs.py
|
||||
index 956d808..8cc65e4 100644
|
||||
--- a/tests/test_posthocs.py
|
||||
+++ b/tests/test_posthocs.py
|
||||
@@ -471,7 +471,7 @@ class TestPosthocs(unittest.TestCase):
|
||||
[2.857818e-06, 1.230888e-05, 1]])
|
||||
|
||||
results = sp.posthoc_wilcoxon(self.df.sort_index(), val_col = 'pulse', group_col = 'kind')
|
||||
- self.assertTrue(np.allclose(results, r_results))
|
||||
+ self.assertTrue(np.allclose(results, r_results, atol=1e-4))
|
||||
|
||||
|
||||
def test_posthoc_scheffe(self):
|
||||
--
|
||||
2.36.1
|
||||
|
||||
@ -1,34 +0,0 @@
|
||||
From 5416ffba3ab01aebab3909400b5a9e847022898e Mon Sep 17 00:00:00 2001
|
||||
From: Maksim Terpilovskii <maximtrp@gmail.com>
|
||||
Date: Thu, 16 Mar 2023 00:20:02 +0300
|
||||
Subject: [PATCH] Update test_posthocs.py
|
||||
|
||||
---
|
||||
tests/test_posthocs.py | 4 ++--
|
||||
1 file changed, 2 insertions(+), 2 deletions(-)
|
||||
|
||||
diff --git a/tests/test_posthocs.py b/tests/test_posthocs.py
|
||||
index 8cc65e4..42ca5f3 100644
|
||||
--- a/tests/test_posthocs.py
|
||||
+++ b/tests/test_posthocs.py
|
||||
@@ -71,7 +71,7 @@ class TestPosthocs(unittest.TestCase):
|
||||
a = splt.sign_plot(x, flat=True, labels=False)
|
||||
with self.assertRaises(ValueError):
|
||||
splt.sign_plot(x.astype(float), flat=True, labels=False)
|
||||
- self.assertTrue(isinstance(a, ma._subplots.Axes))
|
||||
+ self.assertTrue(isinstance(a, ma._axes.Axes))
|
||||
|
||||
def test_sign_plot_nonflat(self):
|
||||
|
||||
@@ -85,7 +85,7 @@ class TestPosthocs(unittest.TestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
splt.sign_plot(x.astype(np.int64), labels=False)
|
||||
|
||||
- self.assertTrue(isinstance(a, ma._subplots.Axes) and isinstance(cbar, mpl.colorbar.ColorbarBase))
|
||||
+ self.assertTrue(isinstance(a, ma._axes.Axes) and isinstance(cbar, mpl.colorbar.ColorbarBase))
|
||||
|
||||
# Outliers tests
|
||||
def test_outliers_iqr(self):
|
||||
--
|
||||
2.36.1
|
||||
|
||||
@ -1,96 +0,0 @@
|
||||
diff --git a/sopel/plugins/handlers.py b/sopel/plugins/handlers.py
|
||||
index 76902aa0..05f0279d 100644
|
||||
--- a/sopel/plugins/handlers.py
|
||||
+++ b/sopel/plugins/handlers.py
|
||||
@@ -46,20 +46,15 @@ from __future__ import absolute_import, division, print_function, unicode_litera
|
||||
|
||||
import imp
|
||||
import importlib
|
||||
+import importlib.util
|
||||
import inspect
|
||||
import itertools
|
||||
import os
|
||||
+import sys
|
||||
|
||||
from sopel import loader
|
||||
from . import exceptions
|
||||
|
||||
-try:
|
||||
- reload = importlib.reload
|
||||
-except AttributeError:
|
||||
- # py2: no reload function
|
||||
- # TODO: imp is deprecated, to be removed when py2 support is dropped
|
||||
- reload = imp.reload
|
||||
-
|
||||
|
||||
class AbstractPluginHandler(object):
|
||||
"""Base class for plugin handlers.
|
||||
@@ -301,7 +296,7 @@ class PyModulePlugin(AbstractPluginHandler):
|
||||
|
||||
This method assumes the plugin is already loaded.
|
||||
"""
|
||||
- self._module = reload(self._module)
|
||||
+ self._module = importlib.reload(self._module)
|
||||
|
||||
def is_loaded(self):
|
||||
return self._module is not None
|
||||
@@ -402,45 +397,31 @@ class PyFilePlugin(PyModulePlugin):
|
||||
|
||||
if good_file:
|
||||
name = os.path.basename(filename)[:-3]
|
||||
- module_type = imp.PY_SOURCE
|
||||
+ spec = importlib.util.spec_from_file_location(
|
||||
+ name,
|
||||
+ filename,
|
||||
+ )
|
||||
elif good_dir:
|
||||
name = os.path.basename(filename)
|
||||
- module_type = imp.PKG_DIRECTORY
|
||||
+ spec = importlib.util.spec_from_file_location(
|
||||
+ name,
|
||||
+ os.path.join(filename, '__init__.py'),
|
||||
+ submodule_search_locations=filename,
|
||||
+ )
|
||||
else:
|
||||
raise exceptions.PluginError('Invalid Sopel plugin: %s' % filename)
|
||||
|
||||
self.filename = filename
|
||||
self.path = filename
|
||||
- self.module_type = module_type
|
||||
+ self.module_spec = spec
|
||||
|
||||
super(PyFilePlugin, self).__init__(name)
|
||||
|
||||
def _load(self):
|
||||
- # The current implementation uses `imp.load_module` to perform the
|
||||
- # load action, which also reloads the module. However, `imp` is
|
||||
- # deprecated in Python 3, so that might need to be changed when the
|
||||
- # support for Python 2 is dropped.
|
||||
- #
|
||||
- # However, the solution for Python 3 is non-trivial, since the
|
||||
- # `importlib` built-in module does not have a similar function,
|
||||
- # therefore requires to dive into its public internals
|
||||
- # (``importlib.machinery`` and ``importlib.util``).
|
||||
- #
|
||||
- # All of that is doable, but represents a lot of work. As long as
|
||||
- # Python 2 is supported, we can keep it for now.
|
||||
- #
|
||||
- # TODO: switch to ``importlib`` when Python2 support is dropped.
|
||||
- if self.module_type == imp.PY_SOURCE:
|
||||
- with open(self.path) as mod:
|
||||
- description = ('.py', 'U', self.module_type)
|
||||
- mod = imp.load_module(self.name, mod, self.path, description)
|
||||
- elif self.module_type == imp.PKG_DIRECTORY:
|
||||
- description = ('', '', self.module_type)
|
||||
- mod = imp.load_module(self.name, None, self.path, description)
|
||||
- else:
|
||||
- raise TypeError('Unsupported module type')
|
||||
-
|
||||
- return mod
|
||||
+ module = importlib.util.module_from_spec(self.module_spec)
|
||||
+ sys.modules[self.name] = module
|
||||
+ self.module_spec.loader.exec_module(module)
|
||||
+ return module
|
||||
|
||||
def get_meta_description(self):
|
||||
"""Retrieve a meta description for the plugin.
|
||||
@ -1,168 +0,0 @@
|
||||
From b30d694a027eb771c02a3db0dee0ca03ccab7377 Mon Sep 17 00:00:00 2001
|
||||
From: Stefan Eissing <stefan@eissing.org>
|
||||
Date: Thu, 28 Mar 2024 11:08:15 +0100
|
||||
Subject: [PATCH] content_encoding: brotli and others, pass through 0-length
|
||||
writes
|
||||
|
||||
- curl's transfer handling may write 0-length chunks at the end of the
|
||||
download with an EOS flag. (HTTP/2 does this commonly)
|
||||
|
||||
- content encoders need to pass-through such a write and not count this
|
||||
as error in case they are finished decoding
|
||||
|
||||
Fixes #13209
|
||||
Fixes #13212
|
||||
Closes #13219
|
||||
---
|
||||
lib/content_encoding.c | 10 +++++-----
|
||||
tests/http/test_02_download.py | 13 +++++++++++++
|
||||
tests/http/testenv/env.py | 7 ++++++-
|
||||
tests/http/testenv/httpd.py | 20 ++++++++++++++++++++
|
||||
4 files changed, 44 insertions(+), 6 deletions(-)
|
||||
|
||||
diff --git a/lib/content_encoding.c b/lib/content_encoding.c
|
||||
index c1abf24e8c027c..8e926dd2ecd5ad 100644
|
||||
--- a/lib/content_encoding.c
|
||||
+++ b/lib/content_encoding.c
|
||||
@@ -300,7 +300,7 @@ static CURLcode deflate_do_write(struct Curl_easy *data,
|
||||
struct zlib_writer *zp = (struct zlib_writer *) writer;
|
||||
z_stream *z = &zp->z; /* zlib state structure */
|
||||
|
||||
- if(!(type & CLIENTWRITE_BODY))
|
||||
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
||||
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
||||
|
||||
/* Set the compressed input when this function is called */
|
||||
@@ -457,7 +457,7 @@ static CURLcode gzip_do_write(struct Curl_easy *data,
|
||||
struct zlib_writer *zp = (struct zlib_writer *) writer;
|
||||
z_stream *z = &zp->z; /* zlib state structure */
|
||||
|
||||
- if(!(type & CLIENTWRITE_BODY))
|
||||
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
||||
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
||||
|
||||
if(zp->zlib_init == ZLIB_INIT_GZIP) {
|
||||
@@ -669,7 +669,7 @@ static CURLcode brotli_do_write(struct Curl_easy *data,
|
||||
CURLcode result = CURLE_OK;
|
||||
BrotliDecoderResult r = BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT;
|
||||
|
||||
- if(!(type & CLIENTWRITE_BODY))
|
||||
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
||||
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
||||
|
||||
if(!bp->br)
|
||||
@@ -762,7 +762,7 @@ static CURLcode zstd_do_write(struct Curl_easy *data,
|
||||
ZSTD_outBuffer out;
|
||||
size_t errorCode;
|
||||
|
||||
- if(!(type & CLIENTWRITE_BODY))
|
||||
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
||||
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
||||
|
||||
if(!zp->decomp) {
|
||||
@@ -916,7 +916,7 @@ static CURLcode error_do_write(struct Curl_easy *data,
|
||||
(void) buf;
|
||||
(void) nbytes;
|
||||
|
||||
- if(!(type & CLIENTWRITE_BODY))
|
||||
+ if(!(type & CLIENTWRITE_BODY) || !nbytes)
|
||||
return Curl_cwriter_write(data, writer->next, type, buf, nbytes);
|
||||
|
||||
failf(data, "Unrecognized content encoding type. "
|
||||
diff --git a/tests/http/test_02_download.py b/tests/http/test_02_download.py
|
||||
index 4db9c9d36e9ed5..395fc862f2f839 100644
|
||||
--- a/tests/http/test_02_download.py
|
||||
+++ b/tests/http/test_02_download.py
|
||||
@@ -394,6 +394,19 @@ def test_02_27_paused_no_cl(self, env: Env, httpd, nghttpx, repeat):
|
||||
r = client.run(args=[url])
|
||||
r.check_exit_code(0)
|
||||
|
||||
+ @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3'])
|
||||
+ def test_02_28_get_compressed(self, env: Env, httpd, nghttpx, repeat, proto):
|
||||
+ if proto == 'h3' and not env.have_h3():
|
||||
+ pytest.skip("h3 not supported")
|
||||
+ count = 1
|
||||
+ urln = f'https://{env.authority_for(env.domain1brotli, proto)}/data-100k?[0-{count-1}]'
|
||||
+ curl = CurlClient(env=env)
|
||||
+ r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[
|
||||
+ '--compressed'
|
||||
+ ])
|
||||
+ r.check_exit_code(code=0)
|
||||
+ r.check_response(count=count, http_status=200)
|
||||
+
|
||||
def check_downloads(self, client, srcfile: str, count: int,
|
||||
complete: bool = True):
|
||||
for i in range(count):
|
||||
diff --git a/tests/http/testenv/env.py b/tests/http/testenv/env.py
|
||||
index a207059dcd57c5..13c5d6bd46ee57 100644
|
||||
--- a/tests/http/testenv/env.py
|
||||
+++ b/tests/http/testenv/env.py
|
||||
@@ -129,10 +129,11 @@ def __init__(self):
|
||||
self.htdocs_dir = os.path.join(self.gen_dir, 'htdocs')
|
||||
self.tld = 'http.curl.se'
|
||||
self.domain1 = f"one.{self.tld}"
|
||||
+ self.domain1brotli = f"brotli.one.{self.tld}"
|
||||
self.domain2 = f"two.{self.tld}"
|
||||
self.proxy_domain = f"proxy.{self.tld}"
|
||||
self.cert_specs = [
|
||||
- CertificateSpec(domains=[self.domain1, 'localhost'], key_type='rsa2048'),
|
||||
+ CertificateSpec(domains=[self.domain1, self.domain1brotli, 'localhost'], key_type='rsa2048'),
|
||||
CertificateSpec(domains=[self.domain2], key_type='rsa2048'),
|
||||
CertificateSpec(domains=[self.proxy_domain, '127.0.0.1'], key_type='rsa2048'),
|
||||
CertificateSpec(name="clientsX", sub_specs=[
|
||||
@@ -376,6 +377,10 @@ def htdocs_dir(self) -> str:
|
||||
def domain1(self) -> str:
|
||||
return self.CONFIG.domain1
|
||||
|
||||
+ @property
|
||||
+ def domain1brotli(self) -> str:
|
||||
+ return self.CONFIG.domain1brotli
|
||||
+
|
||||
@property
|
||||
def domain2(self) -> str:
|
||||
return self.CONFIG.domain2
|
||||
diff --git a/tests/http/testenv/httpd.py b/tests/http/testenv/httpd.py
|
||||
index c04c22699a62c4..b8615875a9a558 100644
|
||||
--- a/tests/http/testenv/httpd.py
|
||||
+++ b/tests/http/testenv/httpd.py
|
||||
@@ -50,6 +50,7 @@ class Httpd:
|
||||
'alias', 'env', 'filter', 'headers', 'mime', 'setenvif',
|
||||
'socache_shmcb',
|
||||
'rewrite', 'http2', 'ssl', 'proxy', 'proxy_http', 'proxy_connect',
|
||||
+ 'brotli',
|
||||
'mpm_event',
|
||||
]
|
||||
COMMON_MODULES_DIRS = [
|
||||
@@ -203,6 +204,7 @@ def _mkpath(self, path):
|
||||
|
||||
def _write_config(self):
|
||||
domain1 = self.env.domain1
|
||||
+ domain1brotli = self.env.domain1brotli
|
||||
creds1 = self.env.get_credentials(domain1)
|
||||
domain2 = self.env.domain2
|
||||
creds2 = self.env.get_credentials(domain2)
|
||||
@@ -285,6 +287,24 @@ def _write_config(self):
|
||||
f'</VirtualHost>',
|
||||
f'',
|
||||
])
|
||||
+ # Alternate to domain1 with BROTLI compression
|
||||
+ conf.extend([ # https host for domain1, h1 + h2
|
||||
+ f'<VirtualHost *:{self.env.https_port}>',
|
||||
+ f' ServerName {domain1brotli}',
|
||||
+ f' Protocols h2 http/1.1',
|
||||
+ f' SSLEngine on',
|
||||
+ f' SSLCertificateFile {creds1.cert_file}',
|
||||
+ f' SSLCertificateKeyFile {creds1.pkey_file}',
|
||||
+ f' DocumentRoot "{self._docs_dir}"',
|
||||
+ f' SetOutputFilter BROTLI_COMPRESS',
|
||||
+ ])
|
||||
+ conf.extend(self._curltest_conf(domain1))
|
||||
+ if domain1 in self._extra_configs:
|
||||
+ conf.extend(self._extra_configs[domain1])
|
||||
+ conf.extend([
|
||||
+ f'</VirtualHost>',
|
||||
+ f'',
|
||||
+ ])
|
||||
conf.extend([ # https host for domain2, no h2
|
||||
f'<VirtualHost *:{self.env.https_port}>',
|
||||
f' ServerName {domain2}',
|
||||
Loading…
Reference in New Issue
Block a user