Skip to content
This repository has been archived by the owner on Jun 15, 2024. It is now read-only.

Commit

Permalink
Implement PipelineStructureConsumer and PipelineStructureSource in de…
Browse files Browse the repository at this point in the history 
…fault persistence mechanism (#135, #6)
  • Loading branch information
@flosell
flosell committed Oct 16, 2016
1 parent 2a45b20 commit 8f92b1b
Show file tree
Hide file tree
Showing 9 changed files with 192 additions and 119 deletions.
2 changes: 1 addition & 1 deletion CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ The official release will have a defined and more stable API. If you are already
## 0.11.0

* Improvements:
* Keeps a history of pipeline structure if persistence component supports it (#131, #6)
* Keeps a history of pipeline structure if persistence component supports it (#131, #6); Implemented for default persistence
* Bug fixes:
* Fix deadlock occurring when steps write a lot of step-results in quick succession and step results are inherited by their parents (as in chaining) (#135)
* API changes:
Expand Down
44 changes: 28 additions & 16 deletions src/clj/lambdacd/internal/default_pipeline_state.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
i.e. what's currently running, what are the results of each step, ..."
(:require [lambdacd.internal.default-pipeline-state-persistence :as persistence]
[clj-time.core :as t]
[lambdacd.internal.pipeline-state :as pipeline-state]
[lambdacd.util :as util]))
[lambdacd.internal.pipeline-state :as old-pipeline-state]
[lambdacd.state.protocols :as protocols]
[lambdacd.util :as util]
[clojure.data :as data]))

(def clean-pipeline-state {})

Expand All @@ -21,36 +23,46 @@
(defn- update-step-result-in-state [build-number step-id new-step-result current-state]
(update-in current-state [build-number step-id] #(update-step-result new-step-result %)))

(defn- truncate-build-history [max-builds state]
(->> state
(sort-by key >)
(take max-builds)
(into {})))
(defn- truncate-build-history [home-dir max-builds state]
(let [new-state (->> state
(sort-by key >)
(take max-builds)
(into {}))
[only-in-old _ _] (data/diff (set (keys state)) (set (keys new-state)))]
(persistence/clean-up-old-builds home-dir only-in-old)
new-state))

(defn- most-recent-build-number-in-state [pipeline-state]
(if-let [current-build-number (last (sort (keys pipeline-state)))]
current-build-number
0))

(defrecord DefaultPipelineState [state-atom home-dir max-builds]
pipeline-state/PipelineStateComponent
(defrecord DefaultPipelineState [state-atom structure-atom home-dir max-builds]
old-pipeline-state/PipelineStateComponent
(update [self build-number step-id new-step-result]
(if (not (nil? state-atom)) ; convenience for tests: if no state exists we just do nothing
(let [new-state (swap! state-atom #(->> %
(update-step-result-in-state build-number step-id new-step-result)
(truncate-build-history max-builds)))]
(persistence/write-build-history home-dir build-number new-state)
(persistence/clean-up-old-history home-dir new-state))))
(truncate-build-history home-dir max-builds)))]
(persistence/write-build-history home-dir build-number new-state))))
(get-all [self]
@state-atom)
(get-internal-state [self]
state-atom)
(next-build-number [self]
(inc (most-recent-build-number-in-state @state-atom))))
(inc (most-recent-build-number-in-state @state-atom)))
protocols/PipelineStructureConsumer
(consume-pipeline-structure [self build-number pipeline-structure-representation]
(swap! structure-atom #(assoc % build-number pipeline-structure-representation))
(persistence/write-pipeline-structure home-dir build-number pipeline-structure-representation))
protocols/PipelineStructureSource
(get-pipeline-structure [self build-number]
(get @structure-atom build-number)))

(defn new-default-pipeline-state [config & {:keys [initial-state-for-testing]}]
(let [state-atom (atom (or initial-state-for-testing (initial-pipeline-state config)))
home-dir (:home-dir config)
(let [home-dir (:home-dir config)
state-atom (atom (or initial-state-for-testing (initial-pipeline-state config)))
structure-atom (atom (persistence/read-pipeline-structures home-dir))
max-builds (or (:max-builds config) Integer/MAX_VALUE)
instance (->DefaultPipelineState state-atom home-dir max-builds)]
instance (->DefaultPipelineState state-atom structure-atom home-dir max-builds)]
instance))
63 changes: 41 additions & 22 deletions src/clj/lambdacd/internal/default_pipeline_state_persistence.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,16 @@
(into {} (map step-result-with-formatted-step-ids->step-result m)))

(defn- build-number-from-path [path]
(util/parse-int (second (re-find #"build-(\d+)" path))))
(util/parse-int (second (re-find #"build-(\d+)" (str path)))))

(defn- build-state-path [dir]
(str dir "/" "build-state.edn"))
(io/file dir "build-state.edn"))

(defn- read-build-edn [dir]
(let [path (build-state-path dir)
build-number (build-number-from-path path)
(defn- pipeline-structure-path [dir]
(io/file dir "pipeline-structure.edn"))

(defn- read-build-edn [path]
(let [build-number (build-number-from-path path)
data-str (slurp path)
state (formatted-step-ids->pipeline-state (dates->clj-times (edn/read-string data-str)))]
{build-number state}))
Expand All @@ -69,25 +71,42 @@
build-dirs (filter #(.startsWith (.getName %) "build-") directories-in-home)]
build-dirs))

(defn- build-dir [home-dir build-number]
(let [result (str home-dir "/" "build-" build-number)]
(.mkdirs (io/file result))
result))

(defn write-build-history [home-dir build-number new-state]
(if home-dir
(let [dir (str home-dir "/" "build-" build-number)
edn-path (build-state-path dir)
build (get new-state build-number)]
(.mkdirs (io/file dir))
(let [dir (build-dir home-dir build-number)
edn-path (build-state-path dir)
build (get new-state build-number)]
(write-build-edn edn-path build))))

(defn read-build-history-from [home-dir]
(let [states (map read-build-edn (build-dirs home-dir))]
(into {} states)))
(defn file-exists? [f]
(.exists f))

(defn clean-up-old-history [home-dir new-state]
(if home-dir
(let [existing-build-dirs (map str (build-dirs home-dir))
expected-build-numbers (keys new-state)
expected-build-dirs (map #(str (io/file home-dir (str "build-" %))) expected-build-numbers)
[only-in-existing _ _] (data/diff (set existing-build-dirs) (set expected-build-dirs))]
(doall (for [old-build-dir only-in-existing]
(do
(log/info "Cleaning up old build directory" old-build-dir)
(fs/delete-dir old-build-dir)))))))
(defn read-build-history-from [home-dir]
(->> (build-dirs home-dir)
(map build-state-path)
(filter file-exists?)
(map read-build-edn)
(into {})))

(defn write-pipeline-structure [home-dir build-number pipeline-structure]
(let [f (pipeline-structure-path (build-dir home-dir build-number))]
(spit f (pr-str pipeline-structure))))

(defn- read-pipeline-structure-edn [f]
{(build-number-from-path f) (edn/read-string (slurp f))})

(defn read-pipeline-structures [home-dir]
(->> (build-dirs home-dir)
(map pipeline-structure-path)
(filter file-exists?)
(map read-pipeline-structure-edn)
(into {})))

(defn clean-up-old-builds [home-dir old-build-numbers]
(doall (map (fn [old-build-number]
(fs/delete-dir (build-dir home-dir old-build-number))) old-build-numbers)))
79 changes: 53 additions & 26 deletions test/clj/lambdacd/internal/default_pipeline_state_persistence_test.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,35 +2,45 @@
(:use [lambdacd.testsupport.test-util])
(:require [clojure.test :refer :all]
[lambdacd.internal.default-pipeline-state-persistence :refer :all]
[lambdacd.presentation.pipeline-structure-test :as pipeline-structure-test]
[clj-time.core :as t]
[lambdacd.util :as utils]
[clojure.java.io :as io]))

(deftest roundtrip-persistence-test
(testing "the standard case"
(let [some-pipeline-state {3 {'(0) {:status :success :most-recent-update-at (t/epoch)}
'(0 1 2) {:status :failure :out "something went wrong"}}}
home-dir (utils/create-temp-dir)]
(write-build-history home-dir 3 some-pipeline-state)
(is (= some-pipeline-state (read-build-history-from home-dir)))))
(testing "that string-keys in a step result are suppored as well (#101)"
(let [some-pipeline-state {3 {'(0) {:status :success :_git-last-seen-revisions {"refs/heads/master" "some-sha"}}}}
home-dir (utils/create-temp-dir)]
(write-build-history home-dir 3 some-pipeline-state)
(is (= some-pipeline-state (read-build-history-from home-dir)))))
(testing "that keyworded values in a step result are suppored as well (#101)"
(let [some-pipeline-state {3 {'(0) {:status :success :v :x}}}
home-dir (utils/create-temp-dir)]
(write-build-history home-dir 3 some-pipeline-state)
(is (= some-pipeline-state (read-build-history-from home-dir)))))
(testing "pipeline-state"
(testing "the standard case"
(let [some-pipeline-state {3 {'(0) {:status :success :most-recent-update-at (t/epoch)}
'(0 1 2) {:status :failure :out "something went wrong"}}}
home-dir (utils/create-temp-dir)]
(write-build-history home-dir 3 some-pipeline-state)
(is (= some-pipeline-state (read-build-history-from home-dir)))))
(testing "that string-keys in a step result are supported as well (#101)"
(let [some-pipeline-state {3 {'(0) {:status :success :_git-last-seen-revisions {"refs/heads/master" "some-sha"}}}}
home-dir (utils/create-temp-dir)]
(write-build-history home-dir 3 some-pipeline-state)
(is (= some-pipeline-state (read-build-history-from home-dir)))))
(testing "that keyworded values in a step result are suppored as well (#101)"
(let [some-pipeline-state {3 {'(0) {:status :success :v :x}}}
home-dir (utils/create-temp-dir)]
(write-build-history home-dir 3 some-pipeline-state)
(is (= some-pipeline-state (read-build-history-from home-dir))))))
(testing "pipeline-structure"
(testing "the standard case"
(let [home-dir (utils/create-temp-dir)
some-pipeline-structure pipeline-structure-test/foo-pipeline-display-representation]
(write-pipeline-structure home-dir 1 some-pipeline-structure)
(write-pipeline-structure home-dir 2 some-pipeline-structure)
(is (= some-pipeline-structure (get (read-pipeline-structures home-dir) 1)))
(is (= some-pipeline-structure (get (read-pipeline-structures home-dir) 2)))))))

(deftest clean-up-old-builds-test
(testing "cleaning up old history"
(testing "that build-directories will be deleted if they no longer exist in the build-state"
(let [some-pipeline-state {0 {'(0) {:status :success}}
1 {'(0) {:status :success}}
2 {'(0) {:status :success}}}
truncated-pipeline-state {1 {'(0) {:status :success}}
(testing "that given build-directories will be deleted"
(let [some-pipeline-state {0 {'(0) {:status :success}}
1 {'(0) {:status :success}}
2 {'(0) {:status :success}}}
home-dir (utils/create-temp-dir)]
home-dir (utils/create-temp-dir)]
(doall (for [build (range 0 3)]
(write-build-history home-dir build some-pipeline-state)))
(doall (for [build (range 0 3)]
Expand All @@ -39,19 +49,36 @@
(is (.exists (io/file home-dir "build-0")))
(is (.exists (io/file home-dir "build-1")))
(is (.exists (io/file home-dir "build-2")))
(clean-up-old-history home-dir truncated-pipeline-state)
(clean-up-old-builds home-dir [0])

(is (not (.exists (io/file home-dir "build-0"))))
(is (.exists (io/file home-dir "build-1")))
(is (.exists (io/file home-dir "build-2")))))
(testing "that it does not clean up things other than build directories"
(let [truncated-pipeline-state {}
home-dir (utils/create-temp-dir)]
(let [home-dir (utils/create-temp-dir)]
(.mkdirs (io/file home-dir "helloworld"))
(is (.exists (io/file home-dir "helloworld")))
(clean-up-old-history home-dir truncated-pipeline-state)
(clean-up-old-builds home-dir [0])
(is (.exists (io/file home-dir "helloworld")))))))

(deftest read-build-history-from-test ; covers only edge-cases that aren't coverd by roundtrip
(testing "that it will return an empty history if no state has been written yet"
(let [home-dir (utils/create-temp-dir)]
(is (= {} (read-build-history-from home-dir)))))
(testing "that it ignores build directories with no build state (e.g. because only structure has been written yet"
(let [home-dir (utils/create-temp-dir)]
(.mkdirs (io/file home-dir "build-1"))
(is (= {} (read-build-history-from home-dir))))))

(deftest read-pipeline-structures-test ; covers only edge-cases that aren't covered by roundtrip
(testing "that it will return an empty data if no state has been written yet"
(let [home-dir (utils/create-temp-dir)]
(is (= {} (read-pipeline-structures home-dir)))))
(testing "that it ignores build directories with no pipeline structure (e.g. because they were created before this feature was available)"
(let [home-dir (utils/create-temp-dir)]
(.mkdirs (io/file home-dir "build-1"))
(is (= {} (read-pipeline-structures home-dir))))))

(defn- roundtrip-date-time [data]
(dates->clj-times
(clj-times->dates data)))
Expand Down
Loading

0 comments on commit 8f92b1b

Please sign in to comment.