Merge pull request #253170 from sternenseemann/hydra-report

maintainers/scripts/haskell: make some hydra perf related workarounds convenient

maintainers/scripts/haskell/hydra-report.hs

@@ -30,7 +30,7 @@ Because step 1) is quite expensive and takes roughly ~5 minutes the result is ca
 {-# OPTIONS_GHC -Wall #-}
 {-# LANGUAGE DataKinds #-}
 
-import Control.Monad (forM_, (<=<))
+import Control.Monad (forM_, forM, (<=<))
 import Control.Monad.Trans (MonadIO (liftIO))
 import Data.Aeson (
    FromJSON,
@@ -108,6 +108,7 @@ newtype JobsetEvalInputs = JobsetEvalInputs {nixpkgs :: Nixpkgs}
 data Eval = Eval
    { id :: Int
    , jobsetevalinputs :: JobsetEvalInputs
+   , builds :: Seq Int
    }
    deriving (Generic, ToJSON, FromJSON, Show)
 
@@ -151,15 +152,20 @@ data Build = Build
    }
    deriving (Generic, ToJSON, FromJSON, Show)
 
+data HydraSlownessWorkaroundFlag = HydraSlownessWorkaround | NoHydraSlownessWorkaround
+data RequestLogsFlag = RequestLogs | NoRequestLogs
+
 main :: IO ()
 main = do
    args <- getArgs
    case args of
-      ["get-report"] -> getBuildReports
+      ["get-report", "--slow"] -> getBuildReports HydraSlownessWorkaround
+      ["get-report"] -> getBuildReports NoHydraSlownessWorkaround
       ["ping-maintainers"] -> printMaintainerPing
-      ["mark-broken-list"] -> printMarkBrokenList
+      ["mark-broken-list", "--no-request-logs"] -> printMarkBrokenList NoRequestLogs
+      ["mark-broken-list"] -> printMarkBrokenList RequestLogs
       ["eval-info"] -> printEvalInfo
-      _ -> putStrLn "Usage: get-report | ping-maintainers | mark-broken-list | eval-info"
+      _ -> putStrLn "Usage: get-report [--slow] | ping-maintainers | mark-broken-list [--no-request-logs] | eval-info"
 
 reportFileName :: IO FilePath
 reportFileName = getXdgDirectory XdgCache "haskell-updates-build-report.json"
@@ -167,18 +173,27 @@ reportFileName = getXdgDirectory XdgCache "haskell-updates-build-report.json"
 showT :: Show a => a -> Text
 showT = Text.pack . show
 
-getBuildReports :: IO ()
-getBuildReports = runReq defaultHttpConfig do
+getBuildReports :: HydraSlownessWorkaroundFlag -> IO ()
+getBuildReports opt = runReq defaultHttpConfig do
    evalMay <- Seq.lookup 0 . evals <$> hydraJSONQuery mempty ["jobset", "nixpkgs", "haskell-updates", "evals"]
    eval@Eval{id} <- maybe (liftIO $ fail "No Evalution found") pure evalMay
    liftIO . putStrLn $ "Fetching evaluation " <> show id <> " from Hydra. This might take a few minutes..."
-   buildReports :: Seq Build <- hydraJSONQuery (responseTimeout 600000000) ["eval", showT id, "builds"]
+   buildReports <- getEvalBuilds opt id
    liftIO do
       fileName <- reportFileName
       putStrLn $ "Finished fetching all builds from Hydra, saving report as " <> fileName
       now <- getCurrentTime
       encodeFile fileName (eval, now, buildReports)
 
+getEvalBuilds :: HydraSlownessWorkaroundFlag -> Int -> Req (Seq Build)
+getEvalBuilds NoHydraSlownessWorkaround id =
+  hydraJSONQuery (responseTimeout 900000000) ["eval", showT id, "builds"]
+getEvalBuilds HydraSlownessWorkaround id = do
+  Eval{builds} <- hydraJSONQuery mempty [ "eval", showT id ]
+  forM builds $ \buildId -> do
+    liftIO $ putStrLn $ "Querying build " <> show buildId
+    hydraJSONQuery mempty [ "build", showT buildId ]
+
 hydraQuery :: HttpResponse a => Proxy a -> Option 'Https -> [Text] -> Req (HttpResponseBody a)
 hydraQuery responseType option query =
    responseBody
@@ -187,7 +202,7 @@ hydraQuery responseType option query =
          (foldl' (/:) (https "hydra.nixos.org") query)
          NoReqBody
          responseType
-         (header "User-Agent" "hydra-report.hs/v1 (nixpkgs;maintainers/scripts/haskell)" <> option)
+         (header "User-Agent" "hydra-report.hs/v1 (nixpkgs;maintainers/scripts/haskell) pls fix https://github.com/NixOS/nixos-org-configurations/issues/270" <> option)
 
 hydraJSONQuery :: FromJSON a => Option 'Https -> [Text] -> Req a
 hydraJSONQuery = hydraQuery jsonResponse
@@ -775,16 +790,20 @@ printMaintainerPing = do
        textBuildSummary = printBuildSummary eval fetchTime buildSum topBrokenRdeps
    Text.putStrLn textBuildSummary
 
-printMarkBrokenList :: IO ()
-printMarkBrokenList = do
+printMarkBrokenList :: RequestLogsFlag -> IO ()
+printMarkBrokenList reqLogs = do
    (_, fetchTime, buildReport) <- readBuildReports
    runReq defaultHttpConfig $ forM_ buildReport \build@Build{job, id} ->
       case (getBuildState build, Text.splitOn "." $ unJobName job) of
          (Failed, ["haskellPackages", name, "x86_64-linux"]) -> do
-            -- Fetch build log from hydra to figure out the cause of the error.
-            build_log <- ByteString.lines <$> hydraPlainQuery ["build", showT id, "nixlog", "1", "raw"]
             -- We use the last probable error cause found in the build log file.
-            let error_message = fromMaybe " failure " $ safeLast $ mapMaybe probableErrorCause build_log
+            error_message <- fromMaybe "failure" <$>
+              case reqLogs of
+                NoRequestLogs -> pure Nothing
+                RequestLogs -> do
+                  -- Fetch build log from hydra to figure out the cause of the error.
+                  build_log <- ByteString.lines <$> hydraPlainQuery ["build", showT id, "nixlog", "1", "raw"]
+                  pure $ safeLast $ mapMaybe probableErrorCause build_log
             liftIO $ putStrLn $ "  - " <> Text.unpack name <> " # " <> error_message <> " in job https://hydra.nixos.org/build/" <> show id <> " at " <> formatTime defaultTimeLocale "%Y-%m-%d" fetchTime
          _ -> pure ()
 

maintainers/scripts/haskell/mark-broken.sh

@@ -10,6 +10,24 @@
 
 set -euo pipefail
 
+do_commit=false
+mark_broken_list_flags=""
+
+for arg in "$@"; do
+    case "$arg" in
+        --do-commit)
+            do_commit=true
+            ;;
+        --no-request-logs)
+            mark_broken_list_flags="$mark_broken_list_flags $arg"
+            ;;
+        *)
+            echo "$0: unknown flag: $arg"
+            exit 100
+            ;;
+    esac
+done
+
 broken_config="pkgs/development/haskell-modules/configuration-hackage2nix/broken.yaml"
 
 tmpfile=$(mktemp)
@@ -17,7 +35,7 @@ trap "rm ${tmpfile}" 0
 
 echo "Remember that you need to manually run 'maintainers/scripts/haskell/hydra-report.hs get-report' sometime before running this script."
 echo "Generating a list of broken builds and displaying for manual confirmation ..."
-maintainers/scripts/haskell/hydra-report.hs mark-broken-list | sort -i > "$tmpfile"
+maintainers/scripts/haskell/hydra-report.hs mark-broken-list $mark_broken_list_flags | sort -i > "$tmpfile"
 
 $EDITOR "$tmpfile"
 
@@ -34,7 +52,7 @@ clear="env -u HOME -u NIXPKGS_CONFIG"
 $clear maintainers/scripts/haskell/regenerate-hackage-packages.sh
 evalline=$(maintainers/scripts/haskell/hydra-report.hs eval-info)
 
-if [[ "${1:-}" == "--do-commit" ]]; then
+if $do_commit; then
 git add $broken_config
 git add pkgs/development/haskell-modules/configuration-hackage2nix/transitive-broken.yaml
 git add pkgs/development/haskell-modules/hackage-packages.nix