mirror of
https://github.com/NixOS/nixpkgs.git
synced 2024-10-31 14:41:27 +00:00
Merge pull request #253170 from sternenseemann/hydra-report
maintainers/scripts/haskell: make some hydra perf related workarounds convenient
This commit is contained in:
commit
964c0d2b18
@ -30,7 +30,7 @@ Because step 1) is quite expensive and takes roughly ~5 minutes the result is ca
|
||||
{-# OPTIONS_GHC -Wall #-}
|
||||
{-# LANGUAGE DataKinds #-}
|
||||
|
||||
import Control.Monad (forM_, (<=<))
|
||||
import Control.Monad (forM_, forM, (<=<))
|
||||
import Control.Monad.Trans (MonadIO (liftIO))
|
||||
import Data.Aeson (
|
||||
FromJSON,
|
||||
@ -108,6 +108,7 @@ newtype JobsetEvalInputs = JobsetEvalInputs {nixpkgs :: Nixpkgs}
|
||||
data Eval = Eval
|
||||
{ id :: Int
|
||||
, jobsetevalinputs :: JobsetEvalInputs
|
||||
, builds :: Seq Int
|
||||
}
|
||||
deriving (Generic, ToJSON, FromJSON, Show)
|
||||
|
||||
@ -151,15 +152,20 @@ data Build = Build
|
||||
}
|
||||
deriving (Generic, ToJSON, FromJSON, Show)
|
||||
|
||||
data HydraSlownessWorkaroundFlag = HydraSlownessWorkaround | NoHydraSlownessWorkaround
|
||||
data RequestLogsFlag = RequestLogs | NoRequestLogs
|
||||
|
||||
main :: IO ()
|
||||
main = do
|
||||
args <- getArgs
|
||||
case args of
|
||||
["get-report"] -> getBuildReports
|
||||
["get-report", "--slow"] -> getBuildReports HydraSlownessWorkaround
|
||||
["get-report"] -> getBuildReports NoHydraSlownessWorkaround
|
||||
["ping-maintainers"] -> printMaintainerPing
|
||||
["mark-broken-list"] -> printMarkBrokenList
|
||||
["mark-broken-list", "--no-request-logs"] -> printMarkBrokenList NoRequestLogs
|
||||
["mark-broken-list"] -> printMarkBrokenList RequestLogs
|
||||
["eval-info"] -> printEvalInfo
|
||||
_ -> putStrLn "Usage: get-report | ping-maintainers | mark-broken-list | eval-info"
|
||||
_ -> putStrLn "Usage: get-report [--slow] | ping-maintainers | mark-broken-list [--no-request-logs] | eval-info"
|
||||
|
||||
reportFileName :: IO FilePath
|
||||
reportFileName = getXdgDirectory XdgCache "haskell-updates-build-report.json"
|
||||
@ -167,18 +173,27 @@ reportFileName = getXdgDirectory XdgCache "haskell-updates-build-report.json"
|
||||
showT :: Show a => a -> Text
|
||||
showT = Text.pack . show
|
||||
|
||||
getBuildReports :: IO ()
|
||||
getBuildReports = runReq defaultHttpConfig do
|
||||
getBuildReports :: HydraSlownessWorkaroundFlag -> IO ()
|
||||
getBuildReports opt = runReq defaultHttpConfig do
|
||||
evalMay <- Seq.lookup 0 . evals <$> hydraJSONQuery mempty ["jobset", "nixpkgs", "haskell-updates", "evals"]
|
||||
eval@Eval{id} <- maybe (liftIO $ fail "No Evalution found") pure evalMay
|
||||
liftIO . putStrLn $ "Fetching evaluation " <> show id <> " from Hydra. This might take a few minutes..."
|
||||
buildReports :: Seq Build <- hydraJSONQuery (responseTimeout 600000000) ["eval", showT id, "builds"]
|
||||
buildReports <- getEvalBuilds opt id
|
||||
liftIO do
|
||||
fileName <- reportFileName
|
||||
putStrLn $ "Finished fetching all builds from Hydra, saving report as " <> fileName
|
||||
now <- getCurrentTime
|
||||
encodeFile fileName (eval, now, buildReports)
|
||||
|
||||
getEvalBuilds :: HydraSlownessWorkaroundFlag -> Int -> Req (Seq Build)
|
||||
getEvalBuilds NoHydraSlownessWorkaround id =
|
||||
hydraJSONQuery (responseTimeout 900000000) ["eval", showT id, "builds"]
|
||||
getEvalBuilds HydraSlownessWorkaround id = do
|
||||
Eval{builds} <- hydraJSONQuery mempty [ "eval", showT id ]
|
||||
forM builds $ \buildId -> do
|
||||
liftIO $ putStrLn $ "Querying build " <> show buildId
|
||||
hydraJSONQuery mempty [ "build", showT buildId ]
|
||||
|
||||
hydraQuery :: HttpResponse a => Proxy a -> Option 'Https -> [Text] -> Req (HttpResponseBody a)
|
||||
hydraQuery responseType option query =
|
||||
responseBody
|
||||
@ -187,7 +202,7 @@ hydraQuery responseType option query =
|
||||
(foldl' (/:) (https "hydra.nixos.org") query)
|
||||
NoReqBody
|
||||
responseType
|
||||
(header "User-Agent" "hydra-report.hs/v1 (nixpkgs;maintainers/scripts/haskell)" <> option)
|
||||
(header "User-Agent" "hydra-report.hs/v1 (nixpkgs;maintainers/scripts/haskell) pls fix https://github.com/NixOS/nixos-org-configurations/issues/270" <> option)
|
||||
|
||||
hydraJSONQuery :: FromJSON a => Option 'Https -> [Text] -> Req a
|
||||
hydraJSONQuery = hydraQuery jsonResponse
|
||||
@ -775,16 +790,20 @@ printMaintainerPing = do
|
||||
textBuildSummary = printBuildSummary eval fetchTime buildSum topBrokenRdeps
|
||||
Text.putStrLn textBuildSummary
|
||||
|
||||
printMarkBrokenList :: IO ()
|
||||
printMarkBrokenList = do
|
||||
printMarkBrokenList :: RequestLogsFlag -> IO ()
|
||||
printMarkBrokenList reqLogs = do
|
||||
(_, fetchTime, buildReport) <- readBuildReports
|
||||
runReq defaultHttpConfig $ forM_ buildReport \build@Build{job, id} ->
|
||||
case (getBuildState build, Text.splitOn "." $ unJobName job) of
|
||||
(Failed, ["haskellPackages", name, "x86_64-linux"]) -> do
|
||||
-- Fetch build log from hydra to figure out the cause of the error.
|
||||
build_log <- ByteString.lines <$> hydraPlainQuery ["build", showT id, "nixlog", "1", "raw"]
|
||||
-- We use the last probable error cause found in the build log file.
|
||||
let error_message = fromMaybe " failure " $ safeLast $ mapMaybe probableErrorCause build_log
|
||||
error_message <- fromMaybe "failure" <$>
|
||||
case reqLogs of
|
||||
NoRequestLogs -> pure Nothing
|
||||
RequestLogs -> do
|
||||
-- Fetch build log from hydra to figure out the cause of the error.
|
||||
build_log <- ByteString.lines <$> hydraPlainQuery ["build", showT id, "nixlog", "1", "raw"]
|
||||
pure $ safeLast $ mapMaybe probableErrorCause build_log
|
||||
liftIO $ putStrLn $ " - " <> Text.unpack name <> " # " <> error_message <> " in job https://hydra.nixos.org/build/" <> show id <> " at " <> formatTime defaultTimeLocale "%Y-%m-%d" fetchTime
|
||||
_ -> pure ()
|
||||
|
||||
|
@ -10,6 +10,24 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
do_commit=false
|
||||
mark_broken_list_flags=""
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--do-commit)
|
||||
do_commit=true
|
||||
;;
|
||||
--no-request-logs)
|
||||
mark_broken_list_flags="$mark_broken_list_flags $arg"
|
||||
;;
|
||||
*)
|
||||
echo "$0: unknown flag: $arg"
|
||||
exit 100
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
broken_config="pkgs/development/haskell-modules/configuration-hackage2nix/broken.yaml"
|
||||
|
||||
tmpfile=$(mktemp)
|
||||
@ -17,7 +35,7 @@ trap "rm ${tmpfile}" 0
|
||||
|
||||
echo "Remember that you need to manually run 'maintainers/scripts/haskell/hydra-report.hs get-report' sometime before running this script."
|
||||
echo "Generating a list of broken builds and displaying for manual confirmation ..."
|
||||
maintainers/scripts/haskell/hydra-report.hs mark-broken-list | sort -i > "$tmpfile"
|
||||
maintainers/scripts/haskell/hydra-report.hs mark-broken-list $mark_broken_list_flags | sort -i > "$tmpfile"
|
||||
|
||||
$EDITOR "$tmpfile"
|
||||
|
||||
@ -34,7 +52,7 @@ clear="env -u HOME -u NIXPKGS_CONFIG"
|
||||
$clear maintainers/scripts/haskell/regenerate-hackage-packages.sh
|
||||
evalline=$(maintainers/scripts/haskell/hydra-report.hs eval-info)
|
||||
|
||||
if [[ "${1:-}" == "--do-commit" ]]; then
|
||||
if $do_commit; then
|
||||
git add $broken_config
|
||||
git add pkgs/development/haskell-modules/configuration-hackage2nix/transitive-broken.yaml
|
||||
git add pkgs/development/haskell-modules/hackage-packages.nix
|
||||
|
Loading…
Reference in New Issue
Block a user