add script to list variance in a csv

mapping_geojson_to_osm_tags/etalab_data/.~lock.all.csv#

@@ -0,0 +1 @@
+,tykayn,spaceship,07.08.2023 15:16,file:///home/tykayn/.config/libreoffice/4;

mapping_geojson_to_osm_tags/etalab_data/.~lock.consolidation-etalab-schema-irve-statique-v-2.2.0-20230728.csv#

@@ -1 +0,0 @@
-,tykayn,spaceship,05.08.2023 21:10,file:///home/tykayn/.config/libreoffice/4;

mapping_geojson_to_osm_tags/etalab_data/.~lock.small.csv#

@@ -0,0 +1 @@
+,tykayn,spaceship,07.08.2023 15:20,file:///home/tykayn/.config/libreoffice/4;

mapping_geojson_to_osm_tags/etalab_data/small.csv

@@ -0,0 +1,4 @@
+nom_amenageur,siren_amenageur,contact_amenageur,nom_operateur,contact_operateur,telephone_operateur,nom_enseigne,id_station_itinerance,id_station_local,nom_station,implantation_station,adresse_station,code_insee_commune,coordonneesXY,nbre_pdc,id_pdc_itinerance,id_pdc_local,puissance_nominale,prise_type_ef,prise_type_2,prise_type_combo_ccs,prise_type_chademo,prise_type_autre,gratuit,paiement_acte,paiement_cb,paiement_autre,tarification,condition_acces,reservation,horaires,accessibilite_pmr,restriction_gabarit,station_deux_roues,raccordement,num_pdl,date_mise_en_service,observations,date_maj,cable_t2_attache,last_modified,datagouv_dataset_id,datagouv_resource_id,datagouv_organization_or_owner,consolidated_longitude,consolidated_latitude,consolidated_code_postal,consolidated_commune,consolidated_is_lon_lat_correct,consolidated_is_code_insee_verified
+ELECTRA,891624884,help@electra.com,ELECTRA,help@electra.com,,ELECTRA,FRELCPBROHI,,Bron - Hôtel Ibis Lyon Est,Station dédiée à la recharge rapide,36 avenue du Doyen Jean Lépine 69500 Bron,69029,"[4.90415400,45.74800500]",4,FRELCE2JW9,,225,false,false,true,false,false,false,true,true,true,,Accès libre,true,24/7,Accessibilité inconnue,Inconnu,false,Direct,N/A,2023-05-04,Télécharger l'application ELECTRA pour réserver et payer sur go-electra.com,2023-08-06,,2023-08-06T03:05:25.841000+00:00,623ca46c13130c3228abd018,e9bb3424-77cd-40ba-8bbd-5a19362d0365,electra,4.904154,45.748005,69500,Bron,True,True
+ELECTRA,891624884,help@electra.com,ELECTRA,help@electra.com,,ELECTRA,FRELCPBLOHM,,Blotzheim - Hôtel Mercure Bâle Mulhouse Aéroport,Station dédiée à la recharge rapide,3 rue de l'industrie 68730 Blotzheim,68042,"[7.50290400,47.60821400]",8,FRELCEXY9P,,225,false,false,true,false,false,false,true,true,true,,Accès libre,true,24/7,Accessibilité inconnue,Inconnu,false,Direct,N/A,2022-09-22,Télécharger l'application ELECTRA pour réserver et payer sur go-electra.com,2023-08-06,,2023-08-06T03:05:25.841000+00:00,623ca46c13130c3228abd018,e9bb3424-77cd-40ba-8bbd-5a19362d0365,electra,7.502904,47.608214,68730,Blotzheim,True,True
+ELECTRA,891624884,help@electra.com,ELECTRA,help@electra.com,,ELECTRA,FRELCPSMCLE,,Saint-Magne-de-Castillon - E.Leclerc,Station dédiée à la recharge rapide,2 chemin de Perrin 33350 Saint-Magne-de-Castillon,33437,"[-0.06340700,44.85401900]",4,FRELCEEEAW,,150,false,false,true,false,false,false,true,true,true,,Accès libre,true,24/7,Accessibilité inconnue,Inconnu,false,Direct,N/A,2023-04-28,Télécharger l'application ELECTRA pour réserver et payer sur go-electra.com,2023-08-06,,2023-08-06T03:05:25.841000+00:00,623ca46c13130c3228abd018,e9bb3424-77cd-40ba-8bbd-5a19362d0365,electra,-0.063407,44.854019,33350,Saint-Magne-de-Castillon,True,True

mapping_geojson_to_osm_tags/make_variance_from_csv.ts

@@ -5,3 +5,111 @@
  faire un nouveau csv qui ne montre que les valeurs uniques pour chacune des colonnes
  et qui compte le nombre de valeurs
  **/
+import utils from './mappings/utils'
+import {parse} from 'csv'
+
+const fs = require('fs')
+
+interface VarianceType {
+    [key: string]: Array<string>
+}
+
+let csv_content = 'variance de dataset\n';
+let separator = ';';
+// let columns_headings = [];
+let data_variance: VarianceType = {};
+
+const inputPath = './etalab_data/small.csv'
+let columns_headings: Array<string> = [];
+let lines_count = 0;
+let longest_variance_count = 0;
+
+console.log('open file ', inputPath)
+fs.readFile(inputPath, function (err: any, fileData: any) {
+
+    if (err) {
+        throw new Error(err)
+    } else {
+        parse(fileData, {columns: false, trim: true}, function (err: any, rows: any) {
+            // Your CSV data is in an array of arrays passed to this callback as rows.
+
+            console.log('line ', lines_count)
+
+            rows.forEach((row: Array<any>) => {
+
+                console.log('ajout de données de la ligne ', lines_count)
+
+                if (lines_count === 0) {
+                    // console.log('elem', row)
+                    row.forEach((value: string) => {
+                        // console.log('value', value)
+                        columns_headings.push(value)
+                        data_variance[value] = []
+                    })
+                    lines_count++
+                } else {
+                    // lignes suivantes
+
+                    let column_index = 0;
+
+
+                    row.forEach((value: string) => {
+                        // dans chaque colonne, vérifier que la valeur n'est pas déjà présente dans les index de variance
+                        // si la valeur est nouvelle, l'ajouter
+                        if (data_variance[columns_headings[column_index]].indexOf(value) < 0) {
+                            data_variance[columns_headings[column_index]].push(value)
+                            if (
+                                data_variance[columns_headings[column_index]].length > longest_variance_count
+                            ) {
+                                longest_variance_count = data_variance[columns_headings[column_index]].length
+                            }
+
+                        }
+                        column_index++
+                    })
+                    lines_count++
+                }
+            })
+
+            console.log('longest_variance_count', longest_variance_count)
+
+            utils.writeFile('variance.csv', writeCSVVariance())
+            // console.log('data_variance', data_variance)
+
+        })
+    }
+
+    console.log('parsing done')
+    // console.log('data_variance', data_variance)
+})
+
+/**
+ * écrit un csv avec les données de variance du dataset donné
+ */
+function writeCSVVariance() {
+
+    let csv_content = 'variance de ' + inputPath + '\n'
+    let columns = Object.keys(data_variance);
+
+    // add headings
+    columns_headings.forEach((heading:string)=>{
+        csv_content = csv_content + separator + heading
+    })
+    csv_content = csv_content + '\n\n'
+    // add content of values
+    for (let ii = 0; ii < longest_variance_count; ii++) {
+        csv_content = csv_content + '\n'
+        columns.forEach((column: any) => {
+            if (ii < data_variance[column].length) {
+
+                let currentValue = data_variance[column][ii]
+                csv_content = csv_content + separator + currentValue
+            } else {
+                csv_content = csv_content + separator
+            }
+        })
+    }
+
+    console.log('csv_content', csv_content)
+    return csv_content;
+}

mapping_geojson_to_osm_tags/mappings/utils.ts

@@ -54,9 +54,9 @@ function writeFile(fileName: string, fileContent: any) {
         'utf8',
         (err) => {
             if (err) {
-                debugLog(`Error writing file: ${err}`)
+                console.log(`Error writing file: ${err}`)
             } else {
-                debugLog(`File ${fileName} is written successfully!`)
+                console.log(`File ${fileName} is written successfully!`)
             }
         }
     )

mapping_geojson_to_osm_tags/osm_output/bornes-irve-filetered-from-etalab-opendata_dep_1.osm

@@ -0,0 +1 @@
+<osm version="0.6" generator="geojsontoosm"><node id="-1" lat="45.635079000000005" lon="4.822158999999999"><tag k="amenity" v="charging_station"/><tag k="operator" v="ELECTRA"/><tag k="owner:ref:FR:SIREN" v="891624884"/><tag k="email" v="help@electra.com"/><tag k="network" v="ELECTRA"/><tag k="socket:type2_combo" v="yes"/><tag k="fee" v="false"/><tag k="authentication:none" v="true"/><tag k="reservation" v="yes"/><tag k="opening_hours" v="24/7"/><tag k="start_date" v="2022-10-27"/></node></osm>

mapping_geojson_to_osm_tags/osm_output/bornes-irve-filetered-from-etalab-opendata_dep_12.osm

@@ -0,0 +1 @@
+<osm version="0.6" generator="geojsontoosm"><node id="-1" lat="45.635079000000005" lon="4.822158999999999"><tag k="amenity" v="charging_station"/><tag k="operator" v="ELECTRA"/><tag k="owner:ref:FR:SIREN" v="891624884"/><tag k="email" v="help@electra.com"/><tag k="network" v="ELECTRA"/><tag k="socket:type2_combo" v="yes"/><tag k="fee" v="false"/><tag k="authentication:none" v="true"/><tag k="reservation" v="yes"/><tag k="opening_hours" v="24/7"/><tag k="start_date" v="2022-10-27"/></node></osm>

mapping_geojson_to_osm_tags/osm_output/bornes-irve-filetered-from-etalab-opendata_dep_6.osm

@@ -0,0 +1 @@
+<osm version="0.6" generator="geojsontoosm"><node id="-1" lat="45.635079000000005" lon="4.822158999999999"><tag k="amenity" v="charging_station"/><tag k="operator" v="ELECTRA"/><tag k="owner:ref:FR:SIREN" v="891624884"/><tag k="email" v="help@electra.com"/><tag k="network" v="ELECTRA"/><tag k="socket:type2_combo" v="yes"/><tag k="fee" v="false"/><tag k="authentication:none" v="true"/><tag k="reservation" v="yes"/><tag k="opening_hours" v="24/7"/><tag k="start_date" v="2022-10-27"/></node></osm>

mapping_geojson_to_osm_tags/osm_output/bornes-irve-filetered-from-etalab-opendata_dep_69.osm

@@ -0,0 +1 @@
+<osm version="0.6" generator="geojsontoosm"><node id="-1" lat="45.635079000000005" lon="4.822158999999999"><tag k="amenity" v="charging_station"/><tag k="operator" v="ELECTRA"/><tag k="owner:ref:FR:SIREN" v="891624884"/><tag k="email" v="help@electra.com"/><tag k="network" v="ELECTRA"/><tag k="socket:type2_combo" v="yes"/><tag k="fee" v="false"/><tag k="authentication:none" v="true"/><tag k="reservation" v="yes"/><tag k="opening_hours" v="24/7"/><tag k="start_date" v="2022-10-27"/></node></osm>