honore-beaugrand/food-data-etl.ipynb

In [77]:

(require '[clojure.data.csv :as csv]
         '[clojure.edn :as edn]
         '[clojure.java.io :as io])

In [14]:

(def data-food-csv "/home/didier/gitlab/honore-beaugrand/resources/data/food-data.csv")

Out [14]:

#'user/data-food-csv

In [125]:

(defn map-values [f m]
  (into {} (for [[k v] m] [k (f v)])))

(defn beautiful-keyword [s]
    (-> s
        (clojure.string/replace "(" "")
        (clojure.string/replace ")" "")
        (clojure.string/replace " " "_")
        clojure.string/lower-case
        keyword))
   
    
(defn csv-data->maps [csv-data]
  (map zipmap
       (->> (first csv-data) ;; First row is the header
            (map beautiful-keyword) ;; Drop if you want string keys instead
            repeat)
    (rest csv-data)))

(defn cast-value [v]
    (cond
         (= v "NULL") nil
         (some? (re-matches #"-?\d+(.\d*)?" v)) (Double/parseDouble v)
         :else v))

(defn read-food-data-from-csv [csv-file-path]
    (with-open [reader (io/reader csv-file-path)]
        (->> reader
             csv/read-csv
             csv-data->maps
             (map #(map-values cast-value %))
             vec))) 

Out [125]:

#'user/read-food-data-from-csv

In [126]:

(def all-data (read-food-data-from-csv data-food-csv))

Out [126]:

#'user/all-data

In [134]:

(def all-data-string (prn-str all-data))
(spit "/home/didier/gitlab/honore-beaugrand/resources/data/food-data.edn" all-data-string)