didier/honore-beaugrand
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

preparation for second course

Browse Source
This commit is contained in:
didier amyot committed 2018-08-26 17:01:35 -04:00
1 parent 549c124119
commit 319ba391d8
5 files changed
+6663 -1
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Cours numero 3.ipynb
+179
View File
@@ -0,0 +1,179 @@
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Class 3. keep going"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## The homework was **way** too hard!!\n",
"\n",
"We needed some more tools..."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"The [concat](https://clojuredocs.org/clojure.core/concat) function is used to concatenate collections togheter."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Homework correction (Part 1)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"[{:name \"class 1\", :url \"https://gitlab.com/didiercrunch/honore-beaugrand/raw/master/resources/data/homework2/class1.edn\"} {:name \"class 2\", :url \"https://gitlab.com/didiercrunch/honore-beaugrand/raw/master/resources/data/homework2/class2.edn\"} {:name \"class 3\", :url \"https://gitlab.com/didiercrunch/honore-beaugrand/raw/master/resources/data/homework2/class3.edn\"}]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"(def school-url \"https://gitlab.com/didiercrunch/honore-beaugrand/raw/master/resources/data/homework2/classes.edn\")\n",
"(def school-data (read-string (slurp school-url)))\n",
"\n",
"school-data"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"([{:name \"boris\", :sex :male, :age 22} {:name \"bob\", :sex :female, :age 25} {:name \"andrea\", :sex :male, :age 33} {:name \"stephanie\", :sex :female, :age 12} {:name \"alexa\", :sex :female, :age 90} {:name \"boris\", :sex :female, :age 12} {:name \"nathasha\", :sex :male, :age 32} {:name \"marco\", :sex :male, :age 30}] [{:name \"enrico\", :sex :male, :age 32} {:name \"raquel\", :sex :female, :age 15} {:name \"alfred\", :sex :male, :age 17} {:name \"kery\", :sex :female, :age 101} {:name \"alexa\", :sex :female, :age 43} {:name \"dominique\", :sex :female, :age 30} {:name \"sofia\", :sex :male, :age 33} {:name \"clory\", :sex :male, :age 45}] [{:name \"Scot\", :sex :male, :age 23} {:name \"Annmarie\", :sex :female, :age 15} {:name \"Jeremy\", :sex :male, :age 17} {:name \"Theodora\", :sex :female, :age 67} {:name \"Andria\", :sex :female, :age 45} {:name \"Kyoko\", :sex :female, :age 39} {:name \"Jeanine\", :sex :male, :age 41} {:name \"Jeanine\", :sex :male, :age 45}])"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"(def class-urls (map :url school-data))\n",
"(def class-data-raw (map slurp class-urls))\n",
"(def class-data (map read-string class-data-raw))\n",
"class-data"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The total number of student in the school is: 24\n"
]
}
],
"source": [
"(def number-of-student-by-class (map count class-data))\n",
"(def number-of-student-in-the-school (apply + number-of-student-by-class))\n",
"(println \"The total number of student in the school is:\" number-of-student-in-the-school)"
]
},
{
"cell_type": "code",
"execution_count": 32,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"#'user/all-student"
]
},
"execution_count": 32,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"(def all-student (apply concat class-data))"
]
},
{
"cell_type": "code",
"execution_count": 29,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The youngest person is aged 12\n",
"The oldest person is aged 101\n"
]
}
],
"source": [
"(def all-ages (map :age all-student))\n",
"\n",
"(def youger-age (apply min all-ages))\n",
"(def older-age (apply max all-ages))\n",
"\n",
"(println \"The youngest person is aged \" youger-age)\n",
"(println \"The oldest person is aged \" older-age)"
]
},
{
"cell_type": "code",
"execution_count": 34,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"the average age is 36.0\n"
]
}
],
"source": [
"(def sum-of-all-ages (apply + all-ages))\n",
"(def average-age (/ sum-of-all-ages number-of-student-in-the-school))\n",
"(println \"the average age is \" (double average-age))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Lein-Clojure",
"language": "clojure",
"name": "lein-clojure"
},
"language_info": {
"file_extension": ".clj",
"mimetype": "text/x-clojure",
"name": "clojure",
"version": "1.9.0"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
food-data-etl.ipynb
+133
View File
@@ -0,0 +1,133 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 77,
"metadata": {},
"outputs": [],
"source": [
"(require '[clojure.data.csv :as csv]\n",
" '[clojure.edn :as edn]\n",
" '[clojure.java.io :as io])"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {
"scrolled": true
},
"outputs": [
{
"data": {
"text/plain": [
"#'user/data-food-csv"
]
},
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"(def data-food-csv \"/home/didier/gitlab/honore-beaugrand/resources/data/food-data.csv\")"
]
},
{
"cell_type": "code",
"execution_count": 125,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"#'user/read-food-data-from-csv"
]
},
"execution_count": 125,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"(defn map-values [f m]\n",
" (into {} (for [[k v] m] [k (f v)])))\n",
"\n",
"(defn beautiful-keyword [s]\n",
" (-> s\n",
" (clojure.string/replace \"(\" \"\")\n",
" (clojure.string/replace \")\" \"\")\n",
" (clojure.string/replace \" \" \"_\")\n",
" clojure.string/lower-case\n",
" keyword))\n",
" \n",
" \n",
"(defn csv-data->maps [csv-data]\n",
" (map zipmap\n",
" (->> (first csv-data) ;; First row is the header\n",
" (map beautiful-keyword) ;; Drop if you want string keys instead\n",
" repeat)\n",
" (rest csv-data)))\n",
"\n",
"(defn cast-value [v]\n",
" (cond\n",
" (= v \"NULL\") nil\n",
" (some? (re-matches #\"-?\\d+(.\\d*)?\" v)) (Double/parseDouble v)\n",
" :else v))\n",
"\n",
"(defn read-food-data-from-csv [csv-file-path]\n",
" (with-open [reader (io/reader csv-file-path)]\n",
" (->> reader\n",
" csv/read-csv\n",
" csv-data->maps\n",
" (map #(map-values cast-value %))\n",
" vec))) \n"
]
},
{
"cell_type": "code",
"execution_count": 126,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"#'user/all-data"
]
},
"execution_count": 126,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"(def all-data (read-food-data-from-csv data-food-csv))"
]
},
{
"cell_type": "code",
"execution_count": 134,
"metadata": {},
"outputs": [],
"source": [
"(def all-data-string (prn-str all-data))\n",
"(spit \"/home/didier/gitlab/honore-beaugrand/resources/data/food-data.edn\" all-data-string)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Lein-Clojure",
"language": "clojure",
"name": "lein-clojure"
},
"language_info": {
"file_extension": ".clj",
"mimetype": "text/x-clojure",
"name": "clojure",
"version": "1.9.0"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
project.clj
+2 -1
View File
@@ -5,7 +5,8 @@
:url "http://www.eclipse.org/legal/epl-v10.html"}
:dependencies [[org.clojure/clojure "1.9.0"]
[cheshire "5.8.0"]
[clj-http "2.3.0"]]
[clj-http "2.3.0"]
[org.clojure/data.csv "0.1.4"]]
:plugins [[lein-jupyter "0.1.16"]]
:main ^:skip-aot honore-beaugrand.core
:target-path "target/%s"
resources/data/food-data.csv
+6348
View File
File diff suppressed because it is too large. Load diff
resources/data/food-data.edn
+1
View File
@@ -0,0 +1 @@
[{:cholesterol_mg 215.0, :database_number 1001.0, :food_name "Butter, salted", :cupper_mg 0.0, :theobromine_mg 0.0, :sucrose_g nil, :phosphorus_mg 24.0, :calories 717.0, :choline_mg 18.8, :alcohol_g 0.0, :fiber_g 0.0, :iron_mg 0.02, :zinc_mg 0.09, :fat_g 81.11, :saturated_fat_g 51.368, :vitamin_b5_mg 0.11, :potasssium_mg 24.0, :calcium_mg 24.0, :riboflavin_b2_mg 0.034, :caffeine_mg 0.0, :lactose_g nil, :manganese_mg 0.0, :vitamin_c_mg 0.0, :thiamin_b1_mg 0.005, :beta_carotene_mcg 158.0, :vitamin_a_iu 2499.0, :glucose_g nil, :vitamin_d2_ergocalciferol__mcg 0.0, :lutein_and_zeaxanthin_mcg 0.0, :water_g 15.87, :lycopene_mcg 0.0, :sodium_mg 643.0, :food_group "Dairy and Egg Products", :niacin_b3_mg 0.042, :sugar_g 0.06, :beta_cryptoxanthin_mcg 0.0, :vitamin_b6_mg 0.003, :maltose_g nil, :alpha_carotene_mcg 0.0, :selenium_mcg 1.0, :fructose_g nil, :carbohydrates_g 0.06, :net_carbs 0.06, :vitamin_d_mcg 0.0, :folate_b9_mg 3.0, :vitamin_e_mg 2.32, :magnesium_mg 2.0, :ash_g 2.11, :protein_g 0.85, :fluoride_mcg 2.8, :retinol_equivalents_mcg 684.0, :galactose_g nil, :vitamin_b12 0.17, :retinol_mcg 671.0, :starch_g nil, :vitamin_d3_cholecalciferol_mcg 0.0} {:cholesterol_mg 225.0, :database_number 1002.0, :food_name "Butter, whipped, with salt", :cupper_mg 0.01, :theobromine_mg 0.0, :sucrose_g nil, :phosphorus_mg 24.0, :calories 718.0, :choline_mg 18.8, :alcohol_g 0.0, :fiber_g 0.0, :iron_mg 0.05, :zinc_mg 0.05, :fat_g 78.3, :saturated_fat_g 45.39, :vitamin_b5_mg 0.097, :potasssium_mg 41.0, :calcium_mg 23.0, :riboflavin_b2_mg 0.064, :caffeine_mg 0.0, :lactose_g nil, :manganese_mg 0.001, :vitamin_c_mg 0.0, :thiamin_b1_mg 0.007, :beta_carotene_mcg 135.0, :vitamin_a_iu 2468.0, :glucose_g nil, :vitamin_d2_ergocalciferol__mcg 0.0, :lutein_and_zeaxanthin_mcg 13.0, :water_g 16.72, :lycopene_mcg 0.0, :sodium_mg 583.0, :food_group "Dairy and Egg Products", :niacin_b3_mg 0.022, :sugar_g 0.06, :beta_cryptoxanthin_mcg 6.0, :vitamin_b6_mg 0.008, :maltose_g nil, :alpha_carotene_mcg 1.0, :selenium_mcg 0.0, :fructose_g nil, :carbohydrates_g 2.87, :net_carbs 2.87, :vitamin_d_mcg 0.0, :folate_b9_mg 4.0, :vitamin_e_mg 1.37, :magnesium_mg 1.0, :ash_g 1.62, :protein_g 0.49, :fluoride_mcg 2.8, :retinol_equivalents_mcg 683.0, :galactose_g nil, :vitamin_b12 0.07, :retinol_mcg 671.0, :starch_g nil, :vitamin_d3_cholecalciferol_mcg 0.0} {:cholesterol_mg 256.0, :database_number 1003.0, :food_name "Butter oil, anhydrous", :cupper_mg 0.001, :theobromine_mg 0.0, :sucrose_g nil, :phosphorus_mg 3.0, :calories 876.0, :choline_mg 22.3, :alcohol_g 0.0, :fiber_g 0.0, :iron_mg 0.0, :zinc_mg 0.01, :fat_g 99.48, :saturated_fat_g 61.924, :vitamin_b5_mg 0.01, :potasssium_mg 5.0, :calcium_mg 4.0, :riboflavin_b2_mg 0.005, :caffeine_mg 0.0, :lactose_g nil, :manganese_mg 0.0, :vitamin_c_mg 0.0, :thiamin_b1_mg 0.001, :beta_carotene_mcg 193.0, :vitamin_a_iu 3069.0, :glucose_g nil, :vitamin_d2_ergocalciferol__mcg 0.0, :lutein_and_zeaxanthin_mcg 0.0, :water_g 0.24, :lycopene_mcg 0.0, :sodium_mg 2.0, :food_group "Dairy and Egg Products", :niacin_b3_mg 0.003, :sugar_g 0.0, :beta_cryptoxanthin_mcg 0.0, :vitamin_b6_mg 0.001, :maltose_g nil, :alpha_carotene_mcg 0.0, :selenium_mcg 0.0, :fructose_g nil, :carbohydrates_g 0.0, :net_carbs 0.0, :vitamin_d_mcg 0.0, :folate_b9_mg 0.0, :vitamin_e_mg 2.8, :magnesium_mg 0.0, :ash_g 0.0, :protein_g 0.28, :fluoride_mcg nil, :retinol_equivalents_mcg 840.0, :galactose_g nil, :vitamin_b12 0.01, :retinol_mcg 824.0, :starch_g nil, :vitamin_d3_cholecalciferol_mcg 0.0} {:cholesterol_mg 75.0, :database_number 1004.0, :food_name "Cheese, blue", :cupper_mg 0.04, :theobromine_mg 0.0, :sucrose_g nil, :phosphorus_mg 387.0, :calories 353.0, :choline_mg 15.4, :alcohol_g 0.0, :fiber_g 0.0, :iron_mg 0.31, :zinc_mg 2.66, :fat_g 28.74, :saturated_fat_g 18.669, :vitamin_b5_mg 1.729, :potasssium_mg 256.0, :calcium_mg 528.0, :riboflavin_b2_mg 0.382, :caffeine_mg 0.0, :lactose_g nil, :manganese_mg 0.009, :vitamin_c_mg 0.0, :thiamin_b1_mg 0.029, :beta_carotene_mcg 74.0, :vitamin_a_iu 721.0, :glucose_g nil, :vitamin_d2_ergocalciferol__mcg nil, :lutein_and_zeaxanthin_mcg 0.0, :water_g 42.41, :lycopene_mcg 0.0, :sodium_mg 1146.0, :food_group "Dairy and Egg Products", :niacin_b3_mg 1.016, :sugar_g 0.5, :beta_cryptoxanthin_mcg 0.0, :vitamin_b6_mg 0.166, :maltose_g nil, :alpha_carotene_mcg 0.0, :selenium_mcg 14.5, :fructose_g nil, :carbohydrates_g 2.34, :net_carbs 2.34, :vitamin_d_mcg 0.5, :folate_b9_mg 36.0, :vitamin_e_mg 0.25, :magnesium_mg 23.0, :ash_g 5.11, :protein_g 21.4, :fluoride_mcg nil, :retinol_equivalents_mcg 198.0, :galactose_g nil, :vitamin_b12 1.22, :retinol_mcg 192.0, :starch_g nil, :vitamin_d3_cholecalciferol_mcg 0.5} {:cholesterol_mg 94.0, :database_number 1005.0, :food_name "Cheese, brick", :cupper_mg 0.024, :theobromine_mg 0.0, :sucrose_g nil, :phosphorus_mg 451.0, :calories 371.0, :choline_mg 15.4, :alcohol_g 0.0, :fiber_g 0.0, :iron_mg 0.43, :zinc_mg 2.6, :fat_g 29.68, :saturated_fat_g 18.764, :vitamin_b5_mg 0.288, :potasssium_mg 136.0, :calciuLine truncated