mirror of
https://github.com/NixOS/nixpkgs.git
synced 2025-02-16 17:14:00 +00:00
hadoop, nixos/hadoop: remove untarDir
This commit is contained in:
illustris
parent
9be640a232
commit
1cd3c804bf
@ -67,16 +67,16 @@ with lib;
|
||||
mapredSiteDefault = mkOption {
|
||||
default = {
|
||||
"mapreduce.framework.name" = "yarn";
|
||||
"yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
|
||||
"mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
|
||||
"mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}/lib/${cfg.package.untarDir}";
|
||||
"yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
|
||||
"mapreduce.map.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
|
||||
"mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=${cfg.package}";
|
||||
};
|
||||
defaultText = literalExpression ''
|
||||
{
|
||||
"mapreduce.framework.name" = "yarn";
|
||||
"yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}";
|
||||
"mapreduce.map.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}";
|
||||
"mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}";
|
||||
"yarn.app.mapreduce.am.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
|
||||
"mapreduce.map.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
|
||||
"mapreduce.reduce.env" = "HADOOP_MAPRED_HOME=''${config.${opt.package}}";
|
||||
}
|
||||
'';
|
||||
type = types.attrsOf types.anything;
|
||||
@ -154,13 +154,13 @@ with lib;
|
||||
};
|
||||
|
||||
log4jProperties = mkOption {
|
||||
default = "${cfg.package}/lib/${cfg.package.untarDir}/etc/hadoop/log4j.properties";
|
||||
default = "${cfg.package}/etc/hadoop/log4j.properties";
|
||||
defaultText = literalExpression ''
|
||||
"''${config.${opt.package}}/lib/''${config.${opt.package}.untarDir}/etc/hadoop/log4j.properties"
|
||||
"''${config.${opt.package}}/etc/hadoop/log4j.properties"
|
||||
'';
|
||||
type = types.path;
|
||||
example = literalExpression ''
|
||||
"''${pkgs.hadoop}/lib/''${pkgs.hadoop.untarDir}/etc/hadoop/log4j.properties";
|
||||
"''${pkgs.hadoop}/etc/hadoop/log4j.properties";
|
||||
'';
|
||||
description = lib.mdDoc "log4j.properties file added to HADOOP_CONF_DIR";
|
||||
};
|
||||
|
||||
@ -160,7 +160,7 @@ in
|
||||
umount /run/wrappers/yarn-nodemanager/cgroup/cpu || true
|
||||
rm -rf /run/wrappers/yarn-nodemanager/ || true
|
||||
mkdir -p /run/wrappers/yarn-nodemanager/{bin,etc/hadoop,cgroup/cpu}
|
||||
cp ${cfg.package}/lib/${cfg.package.untarDir}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/
|
||||
cp ${cfg.package}/bin/container-executor /run/wrappers/yarn-nodemanager/bin/
|
||||
chgrp hadoop /run/wrappers/yarn-nodemanager/bin/container-executor
|
||||
chmod 6050 /run/wrappers/yarn-nodemanager/bin/container-executor
|
||||
cp ${hadoopConf}/container-executor.cfg /run/wrappers/yarn-nodemanager/etc/hadoop/
|
||||
|
||||
@ -249,7 +249,7 @@ import ../make-test-python.nix ({ package, ... }: {
|
||||
assert "standby" in client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState")
|
||||
client.succeed("sudo -u yarn yarn rmadmin -getAllServiceState | systemd-cat")
|
||||
|
||||
assert "Estimated value of Pi is" in client.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~lib/hadoop-*/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")
|
||||
assert "Estimated value of Pi is" in client.succeed("HADOOP_USER_NAME=hdfs yarn jar $(readlink $(which yarn) | sed -r 's~bin/yarn~share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar~g') pi 2 10")
|
||||
assert "SUCCEEDED" in client.succeed("yarn application -list -appStates FINISHED")
|
||||
'';
|
||||
})
|
||||
|
||||
@ -29,11 +29,11 @@ assert elem stdenv.system [ "x86_64-linux" "x86_64-darwin" "aarch64-linux" "aarc
|
||||
|
||||
let
|
||||
common = {
|
||||
pname, platformAttrs, untarDir ? "${pname}-${version}", jdk
|
||||
, nativeLibs ? [ ], libPatchesGenerator ? (_: ""), tests
|
||||
pname, platformAttrs, jdk, nativeLibs ? [ ]
|
||||
, libPatchesGenerator ? (_: ""), tests
|
||||
}:
|
||||
stdenv.mkDerivation (finalAttrs: {
|
||||
inherit pname jdk untarDir;
|
||||
inherit pname jdk;
|
||||
libPatches = libPatchesGenerator finalAttrs;
|
||||
version = platformAttrs.${stdenv.system}.version or (throw "Unsupported system: ${stdenv.system}");
|
||||
src = fetchurl {
|
||||
@ -51,29 +51,29 @@ let
|
||||
};
|
||||
|
||||
nativeBuildInputs = [ makeWrapper ]
|
||||
++ optionals (stdenv.isLinux && (nativeLibs != [ ] || (libPatches finalAttrs) != "")) [ autoPatchelfHook ];
|
||||
++ optionals (stdenv.isLinux && (nativeLibs != [ ] || (libPatchesGenerator finalAttrs) != "")) [ autoPatchelfHook ];
|
||||
buildInputs = [ openssl ] ++ nativeLibs;
|
||||
|
||||
installPhase = ''
|
||||
mkdir -p $out/{lib/${finalAttrs.untarDir}/conf,bin,lib}
|
||||
mv * $out/lib/${finalAttrs.untarDir}
|
||||
mkdir $out
|
||||
mv * $out/
|
||||
'' + optionalString stdenv.isLinux ''
|
||||
for n in $(find ${finalAttrs.containerExecutor}/bin -type f); do
|
||||
ln -sf "$n" $out/lib/${finalAttrs.untarDir}/bin
|
||||
ln -sf "$n" $out/bin
|
||||
done
|
||||
'' + ''
|
||||
for n in $(find $out/lib/${finalAttrs.untarDir}/bin -type f ! -name "*.*"); do
|
||||
makeWrapper "$n" "$out/bin/$(basename $n)"\
|
||||
for n in $(find $out/bin -type f ! -name "*.*"); do
|
||||
wrapProgram "$n"\
|
||||
--set-default JAVA_HOME ${finalAttrs.jdk.home}\
|
||||
--set-default HADOOP_HOME $out/lib/${finalAttrs.untarDir}\
|
||||
--set-default HADOOP_HOME $out/\
|
||||
--run "test -d /etc/hadoop-conf && export HADOOP_CONF_DIR=\''${HADOOP_CONF_DIR-'/etc/hadoop-conf/'}"\
|
||||
--set-default HADOOP_CONF_DIR $out/lib/${finalAttrs.untarDir}/etc/hadoop/\
|
||||
--set-default HADOOP_CONF_DIR $out/etc/hadoop/\
|
||||
--prefix PATH : "${makeBinPath [ bash coreutils which]}"\
|
||||
--prefix JAVA_LIBRARY_PATH : "${makeLibraryPath finalAttrs.buildInputs}"
|
||||
done
|
||||
'' + optionalString sparkSupport ''
|
||||
# Add the spark shuffle service jar to YARN
|
||||
cp ${spark.src}/yarn/spark-${spark.version}-yarn-shuffle.jar $out/lib/${finalAttrs.untarDir}/share/hadoop/yarn/
|
||||
cp ${spark.src}/yarn/spark-${spark.version}-yarn-shuffle.jar $out/share/hadoop/yarn/
|
||||
'' + (finalAttrs.libPatches);
|
||||
|
||||
passthru = { inherit tests; };
|
||||
@ -101,24 +101,24 @@ let
|
||||
});
|
||||
nativeLibs = [ stdenv.cc.cc.lib protobuf zlib snappy libtirpc ];
|
||||
libPatchesGenerator = finalAttrs: (''
|
||||
ln -s ${getLib cyrus_sasl}/lib/libsasl2.so $out/lib/${finalAttrs.untarDir}/lib/native/libsasl2.so.2
|
||||
ln -s ${getLib openssl}/lib/libcrypto.so $out/lib/${finalAttrs.untarDir}/lib/native/
|
||||
ln -s ${getLib zlib}/lib/libz.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/
|
||||
ln -s ${getLib zstd}/lib/libzstd.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/
|
||||
ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/${finalAttrs.untarDir}/lib/native/
|
||||
ln -s ${getLib cyrus_sasl}/lib/libsasl2.so $out/lib/native/libsasl2.so.2
|
||||
ln -s ${getLib openssl}/lib/libcrypto.so $out/lib/native/
|
||||
ln -s ${getLib zlib}/lib/libz.so.1 $out/lib/native/
|
||||
ln -s ${getLib zstd}/lib/libzstd.so.1 $out/lib/native/
|
||||
ln -s ${getLib bzip2}/lib/libbz2.so.1 $out/lib/native/
|
||||
'' + optionalString stdenv.isLinux ''
|
||||
# libjvm.so for Java >=11
|
||||
patchelf --add-rpath ${finalAttrs.jdk.home}/lib/server $out/lib/${finalAttrs.untarDir}/lib/native/libnativetask.so.1.0.0
|
||||
patchelf --add-rpath ${finalAttrs.jdk.home}/lib/server $out/lib/native/libnativetask.so.1.0.0
|
||||
# Java 8 has libjvm.so at a different path
|
||||
patchelf --add-rpath ${finalAttrs.jdk.home}/jre/lib/amd64/server $out/lib/${finalAttrs.untarDir}/lib/native/libnativetask.so.1.0.0
|
||||
patchelf --add-rpath ${finalAttrs.jdk.home}/jre/lib/amd64/server $out/lib/native/libnativetask.so.1.0.0
|
||||
# NixOS/nixpkgs#193370
|
||||
# This workaround is needed to use protobuf 3.19
|
||||
# for hadoop 3.3
|
||||
patchelf --replace-needed libprotobuf.so.18 libprotobuf.so $out/lib/${finalAttrs.untarDir}/lib/native/libhdfspp.so
|
||||
patchelf --replace-needed libprotobuf.so.18 libprotobuf.so $out/lib/native/libhdfspp.so
|
||||
# for hadoop 3.2
|
||||
patchelf --replace-needed libprotobuf.so.8 libprotobuf.so $out/lib/${finalAttrs.untarDir}/lib/native/libhdfspp.so
|
||||
patchelf --replace-needed libprotobuf.so.8 libprotobuf.so $out/lib/native/libhdfspp.so
|
||||
patchelf --replace-needed libcrypto.so.1.1 libcrypto.so \
|
||||
$out/lib/${finalAttrs.untarDir}/lib/native/{libhdfspp.so.0.1.0,examples/{pipes-sort,wordcount-nopipe,wordcount-part,wordcount-simple}}
|
||||
$out/lib/native/{libhdfspp.so.0.1.0,examples/{pipes-sort,wordcount-nopipe,wordcount-part,wordcount-simple}}
|
||||
'');
|
||||
in
|
||||
{
|
||||
@ -138,7 +138,6 @@ in
|
||||
};
|
||||
aarch64-darwin = aarch64-linux;
|
||||
};
|
||||
untarDir = "${pname}-${platformAttrs.${stdenv.system}.version}";
|
||||
jdk = jdk11_headless;
|
||||
inherit nativeLibs libPatchesGenerator;
|
||||
# TODO: Package and add Intel Storage Acceleration Library
|
||||
|
||||
Loading…
Reference in New Issue
Block a user