preparation for second course

Cours numero 3.ipynb

@@ -0,0 +1,179 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Class 3.  keep going"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## The homework was **way** too hard!!\n",
+    "\n",
+    "We needed some more tools..."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The [concat](https://clojuredocs.org/clojure.core/concat) function is used to concatenate collections togheter."
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Homework correction (Part 1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{:name \"class 1\", :url \"https://gitlab.com/didiercrunch/honore-beaugrand/raw/master/resources/data/homework2/class1.edn\"} {:name \"class 2\", :url \"https://gitlab.com/didiercrunch/honore-beaugrand/raw/master/resources/data/homework2/class2.edn\"} {:name \"class 3\", :url \"https://gitlab.com/didiercrunch/honore-beaugrand/raw/master/resources/data/homework2/class3.edn\"}]"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "(def school-url \"https://gitlab.com/didiercrunch/honore-beaugrand/raw/master/resources/data/homework2/classes.edn\")\n",
+    "(def school-data (read-string (slurp school-url)))\n",
+    "\n",
+    "school-data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "([{:name \"boris\", :sex :male, :age 22} {:name \"bob\", :sex :female, :age 25} {:name \"andrea\", :sex :male, :age 33} {:name \"stephanie\", :sex :female, :age 12} {:name \"alexa\", :sex :female, :age 90} {:name \"boris\", :sex :female, :age 12} {:name \"nathasha\", :sex :male, :age 32} {:name \"marco\", :sex :male, :age 30}] [{:name \"enrico\", :sex :male, :age 32} {:name \"raquel\", :sex :female, :age 15} {:name \"alfred\", :sex :male, :age 17} {:name \"kery\", :sex :female, :age 101} {:name \"alexa\", :sex :female, :age 43} {:name \"dominique\", :sex :female, :age 30} {:name \"sofia\", :sex :male, :age 33} {:name \"clory\", :sex :male, :age 45}] [{:name \"Scot\", :sex :male, :age 23} {:name \"Annmarie\", :sex :female, :age 15} {:name \"Jeremy\", :sex :male, :age 17} {:name \"Theodora\", :sex :female, :age 67} {:name \"Andria\", :sex :female, :age 45} {:name \"Kyoko\", :sex :female, :age 39} {:name \"Jeanine\", :sex :male, :age 41} {:name \"Jeanine\", :sex :male, :age 45}])"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "(def class-urls (map :url school-data))\n",
+    "(def class-data-raw (map slurp class-urls))\n",
+    "(def class-data (map read-string class-data-raw))\n",
+    "class-data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The total number of student in the school is: 24\n"
+     ]
+    }
+   ],
+   "source": [
+    "(def number-of-student-by-class (map count class-data))\n",
+    "(def number-of-student-in-the-school (apply + number-of-student-by-class))\n",
+    "(println \"The total number of student in the school is:\" number-of-student-in-the-school)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "#'user/all-student"
+      ]
+     },
+     "execution_count": 32,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "(def all-student (apply concat class-data))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The youngest person is aged  12\n",
+      "The oldest person is aged  101\n"
+     ]
+    }
+   ],
+   "source": [
+    "(def all-ages (map :age all-student))\n",
+    "\n",
+    "(def youger-age (apply min all-ages))\n",
+    "(def older-age (apply max all-ages))\n",
+    "\n",
+    "(println \"The youngest person is aged \" youger-age)\n",
+    "(println \"The oldest person is aged \" older-age)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "the average age is  36.0\n"
+     ]
+    }
+   ],
+   "source": [
+    "(def sum-of-all-ages (apply + all-ages))\n",
+    "(def average-age (/ sum-of-all-ages number-of-student-in-the-school))\n",
+    "(println \"the average age is \" (double average-age))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Lein-Clojure",
+   "language": "clojure",
+   "name": "lein-clojure"
+  },
+  "language_info": {
+   "file_extension": ".clj",
+   "mimetype": "text/x-clojure",
+   "name": "clojure",
+   "version": "1.9.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

food-data-etl.ipynb

@@ -0,0 +1,133 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 77,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "(require '[clojure.data.csv :as csv]\n",
+    "         '[clojure.edn :as edn]\n",
+    "         '[clojure.java.io :as io])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "#'user/data-food-csv"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "(def data-food-csv \"/home/didier/gitlab/honore-beaugrand/resources/data/food-data.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 125,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "#'user/read-food-data-from-csv"
+      ]
+     },
+     "execution_count": 125,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "(defn map-values [f m]\n",
+    "  (into {} (for [[k v] m] [k (f v)])))\n",
+    "\n",
+    "(defn beautiful-keyword [s]\n",
+    "    (-> s\n",
+    "        (clojure.string/replace \"(\" \"\")\n",
+    "        (clojure.string/replace \")\" \"\")\n",
+    "        (clojure.string/replace \" \" \"_\")\n",
+    "        clojure.string/lower-case\n",
+    "        keyword))\n",
+    "   \n",
+    "    \n",
+    "(defn csv-data->maps [csv-data]\n",
+    "  (map zipmap\n",
+    "       (->> (first csv-data) ;; First row is the header\n",
+    "            (map beautiful-keyword) ;; Drop if you want string keys instead\n",
+    "            repeat)\n",
+    "    (rest csv-data)))\n",
+    "\n",
+    "(defn cast-value [v]\n",
+    "    (cond\n",
+    "         (= v \"NULL\") nil\n",
+    "         (some? (re-matches #\"-?\\d+(.\\d*)?\" v)) (Double/parseDouble v)\n",
+    "         :else v))\n",
+    "\n",
+    "(defn read-food-data-from-csv [csv-file-path]\n",
+    "    (with-open [reader (io/reader csv-file-path)]\n",
+    "        (->> reader\n",
+    "             csv/read-csv\n",
+    "             csv-data->maps\n",
+    "             (map #(map-values cast-value %))\n",
+    "             vec))) \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 126,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "#'user/all-data"
+      ]
+     },
+     "execution_count": 126,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "(def all-data (read-food-data-from-csv data-food-csv))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 134,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "(def all-data-string (prn-str all-data))\n",
+    "(spit \"/home/didier/gitlab/honore-beaugrand/resources/data/food-data.edn\" all-data-string)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Lein-Clojure",
+   "language": "clojure",
+   "name": "lein-clojure"
+  },
+  "language_info": {
+   "file_extension": ".clj",
+   "mimetype": "text/x-clojure",
+   "name": "clojure",
+   "version": "1.9.0"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

project.clj

@@ -5,7 +5,8 @@
             :url "http://www.eclipse.org/legal/epl-v10.html"}
   :dependencies [[org.clojure/clojure "1.9.0"]
                  [cheshire "5.8.0"]
-                 [clj-http "2.3.0"]]
+                 [clj-http "2.3.0"]
+                 [org.clojure/data.csv "0.1.4"]]
   :plugins [[lein-jupyter "0.1.16"]]
   :main ^:skip-aot honore-beaugrand.core
   :target-path "target/%s"

resources/data/food-data.edn

@@ -0,0 +1 @@
+[{:cholesterol_mg 215.0, :database_number 1001.0, :food_name "Butter, salted", :cupper_mg 0.0, :theobromine_mg 0.0, :sucrose_g nil, :phosphorus_mg 24.0, :calories 717.0, :choline_mg 18.8, :alcohol_g 0.0, :fiber_g 0.0, :iron_mg 0.02, :zinc_mg 0.09, :fat_g 81.11, :saturated_fat_g 51.368, :vitamin_b5_mg 0.11, :potasssium_mg 24.0, :calcium_mg 24.0, :riboflavin_b2_mg 0.034, :caffeine_mg 0.0, :lactose_g nil, :manganese_mg 0.0, :vitamin_c_mg 0.0, :thiamin_b1_mg 0.005, :beta_carotene_mcg 158.0, :vitamin_a_iu 2499.0, :glucose_g nil, :vitamin_d2_ergocalciferol__mcg 0.0, :lutein_and_zeaxanthin_mcg 0.0, :water_g 15.87, :lycopene_mcg 0.0, :sodium_mg 643.0, :food_group "Dairy and Egg Products", :niacin_b3_mg 0.042, :sugar_g 0.06, :beta_cryptoxanthin_mcg 0.0, :vitamin_b6_mg 0.003, :maltose_g nil, :alpha_carotene_mcg 0.0, :selenium_mcg 1.0, :fructose_g nil, :carbohydrates_g 0.06, :net_carbs 0.06, :vitamin_d_mcg 0.0, :folate_b9_mg 3.0, :vitamin_e_mg 2.32, :magnesium_mg 2.0, :ash_g 2.11, :protein_g 0.85, :fluoride_mcg 2.8, :retinol_equivalents_mcg 684.0, :galactose_g nil, :vitamin_b12 0.17, :retinol_mcg 671.0, :starch_g nil, :vitamin_d3_cholecalciferol_mcg 0.0} {:cholesterol_mg 225.0, :database_number 1002.0, :food_name "Butter, whipped, with salt", :cupper_mg 0.01, :theobromine_mg 0.0, :sucrose_g nil, :phosphorus_mg 24.0, :calories 718.0, :choline_mg 18.8, :alcohol_g 0.0, :fiber_g 0.0, :iron_mg 0.05, :zinc_mg 0.05, :fat_g 78.3, :saturated_fat_g 45.39, :vitamin_b5_mg 0.097, :potasssium_mg 41.0, :calcium_mg 23.0, :riboflavin_b2_mg 0.064, :caffeine_mg 0.0, :lactose_g nil, :manganese_mg 0.001, :vitamin_c_mg 0.0, :thiamin_b1_mg 0.007, :beta_carotene_mcg 135.0, :vitamin_a_iu 2468.0, :glucose_g nil, :vitamin_d2_ergocalciferol__mcg 0.0, :lutein_and_zeaxanthin_mcg 13.0, :water_g 16.72, :lycopene_mcg 0.0, :sodium_mg 583.0, :food_group "Dairy and Egg Products", :niacin_b3_mg 0.022, :sugar_g 0.06, :beta_cryptoxanthin_mcg 6.0, :vitamin_b6_mg 0.008, :maltose_g nil, :alpha_carotene_mcg 1.0, :selenium_mcg 0.0, :fructose_g nil, :carbohydrates_g 2.87, :net_carbs 2.87, :vitamin_d_mcg 0.0, :folate_b9_mg 4.0, :vitamin_e_mg 1.37, :magnesium_mg 1.0, :ash_g 1.62, :protein_g 0.49, :fluoride_mcg 2.8, :retinol_equivalents_mcg 683.0, :galactose_g nil, :vitamin_b12 0.07, :retinol_mcg 671.0, :starch_g nil, :vitamin_d3_cholecalciferol_mcg 0.0} {:cholesterol_mg 256.0, :database_number 1003.0, :food_name "Butter oil, anhydrous", :cupper_mg 0.001, :theobromine_mg 0.0, :sucrose_g nil, :phosphorus_mg 3.0, :calories 876.0, :choline_mg 22.3, :alcohol_g 0.0, :fiber_g 0.0, :iron_mg 0.0, :zinc_mg 0.01, :fat_g 99.48, :saturated_fat_g 61.924, :vitamin_b5_mg 0.01, :potasssium_mg 5.0, :calcium_mg 4.0, :riboflavin_b2_mg 0.005, :caffeine_mg 0.0, :lactose_g nil, :manganese_mg 0.0, :vitamin_c_mg 0.0, :thiamin_b1_mg 0.001, :beta_carotene_mcg 193.0, :vitamin_a_iu 3069.0, :glucose_g nil, :vitamin_d2_ergocalciferol__mcg 0.0, :lutein_and_zeaxanthin_mcg 0.0, :water_g 0.24, :lycopene_mcg 0.0, :sodium_mg 2.0, :food_group "Dairy and Egg Products", :niacin_b3_mg 0.003, :sugar_g 0.0, :beta_cryptoxanthin_mcg 0.0, :vitamin_b6_mg 0.001, :maltose_g nil, :alpha_carotene_mcg 0.0, :selenium_mcg 0.0, :fructose_g nil, :carbohydrates_g 0.0, :net_carbs 0.0, :vitamin_d_mcg 0.0, :folate_b9_mg 0.0, :vitamin_e_mg 2.8, :magnesium_mg 0.0, :ash_g 0.0, :protein_g 0.28, :fluoride_mcg nil, :retinol_equivalents_mcg 840.0, :galactose_g nil, :vitamin_b12 0.01, :retinol_mcg 824.0, :starch_g nil, :vitamin_d3_cholecalciferol_mcg 0.0} {:cholesterol_mg 75.0, :database_number 1004.0, :food_name "Cheese, blue", :cupper_mg 0.04, :theobromine_mg 0.0, :sucrose_g nil, :phosphorus_mg 387.0, :calories 353.0, :choline_mg 15.4, :alcohol_g 0.0, :fiber_g 0.0, :iron_mg 0.31, :zinc_mg 2.66, :fat_g 28.74, :saturated_fat_g 18.669, :vitamin_b5_mg 1.729, :potasssium_mg 256.0, :calcium_mg 528.0, :riboflavin_b2_mg 0.382, :caffeine_mg 0.0, :lactose_g nil, :manganese_mg 0.009, :vitamin_c_mg 0.0, :thiamin_b1_mg 0.029, :beta_carotene_mcg 74.0, :vitamin_a_iu 721.0, :glucose_g nil, :vitamin_d2_ergocalciferol__mcg nil, :lutein_and_zeaxanthin_mcg 0.0, :water_g 42.41, :lycopene_mcg 0.0, :sodium_mg 1146.0, :food_group "Dairy and Egg Products", :niacin_b3_mg 1.016, :sugar_g 0.5, :beta_cryptoxanthin_mcg 0.0, :vitamin_b6_mg 0.166, :maltose_g nil, :alpha_carotene_mcg 0.0, :selenium_mcg 14.5, :fructose_g nil, :carbohydrates_g 2.34, :net_carbs 2.34, :vitamin_d_mcg 0.5, :folate_b9_mg 36.0, :vitamin_e_mg 0.25, :magnesium_mg 23.0, :ash_g 5.11, :protein_g 21.4, :fluoride_mcg nil, :retinol_equivalents_mcg 198.0, :galactose_g nil, :vitamin_b12 1.22, :retinol_mcg 192.0, :starch_g nil, :vitamin_d3_cholecalciferol_mcg 0.5} {:cholesterol_mg 94.0, :database_number 1005.0, :food_name "Cheese, brick", :cupper_mg 0.024, :theobromine_mg 0.0, :sucrose_g nil, :phosphorus_mg 451.0, :calories 371.0, :choline_mg 15.4, :alcohol_g 0.0, :fiber_g 0.0, :iron_mg 0.43, :zinc_mg 2.6, :fat_g 29.68, :saturated_fat_g 18.764, :vitamin_b5_mg 0.288, :potasssium_mg 136.0, :calciuLine truncated