From 5cb3cc02b148ab7a1551a0366b790427f9e59f62 Mon Sep 17 00:00:00 2001 From: Tykayn Date: Sun, 30 Jul 2023 18:37:07 +0200 Subject: [PATCH] refacto to use mapping engine class --- .../convert_to_osm_tags.mjs | 348 +++++++++--------- mapping_geojson_to_osm_tags/dataset_info.md | 2 +- .../mappings/configIRVE.js | 158 ++++---- .../mappings/engine.js | 75 ++++ mapping_geojson_to_osm_tags/mappings/utils.js | 18 + 5 files changed, 357 insertions(+), 244 deletions(-) create mode 100644 mapping_geojson_to_osm_tags/mappings/engine.js create mode 100644 mapping_geojson_to_osm_tags/mappings/utils.js diff --git a/mapping_geojson_to_osm_tags/convert_to_osm_tags.mjs b/mapping_geojson_to_osm_tags/convert_to_osm_tags.mjs index 6c137dbf..5820e96f 100644 --- a/mapping_geojson_to_osm_tags/convert_to_osm_tags.mjs +++ b/mapping_geojson_to_osm_tags/convert_to_osm_tags.mjs @@ -4,226 +4,238 @@ import fs from 'node-fs' import minimist from 'minimist' import mappingConfigIRVE from 'mappings/configIRVE' +import mapping_engine from "./mappings/engine"; +import {debugLog} from "./mappings/utils"; + +// let debugLog = custom_utils.debugLog + +let mapping_engine = new mapping_engine(mappingConfigIRVE); +let use_mappping_engine = false; let mini_arguments = minimist(process.argv.slice(2)) -let show_debug = 0 + +// let sourceFileChargemapJson = './chargemap_data/hurepoix.json' +let sourceFilePathGeoJson = './etalab_data/latest.json' + // wip filter let filterOnBoundingBox = true filterOnBoundingBox = false let boundingBoxCoordinates = { - xMin: 1.91, - xMax: 2.38, - yMin: 48.7, - yMax: 48.4, + xMin: 1.91, + xMax: 2.38, + yMin: 48.7, + yMax: 48.4, } let filterCoordinates = true -let enableFilterOnDepartment = true +let enable_filter_on_department = true let filterDepartment = 91 if (mini_arguments['department']) { - filterDepartment = mini_arguments['department'] + filterDepartment = mini_arguments['department'] +} +if (mini_arguments['source']) { + sourceFilePathGeoJson = mini_arguments['source'] } let filterZipCode = new RegExp(`^${filterDepartment}`) let filterZipCodeAdresse = new RegExp(` ${filterDepartment}`) let filteredName = '' -if (enableFilterOnDepartment) { - filteredName = '_filtered_zipcode_' + filterDepartment +if (enable_filter_on_department) { + filteredName = '_filtered_zipcode_' + filterDepartment } else if (filterOnBoundingBox) { - filteredName = '_filtered_bbox_' + boundingBoxCoordinates.xMin + '-' + boundingBoxCoordinates.xMax + '_' + boundingBoxCoordinates.yMin + '-' + boundingBoxCoordinates.yMax + filteredName = '_filtered_bbox_' + boundingBoxCoordinates.xMin + '-' + boundingBoxCoordinates.xMax + '_' + boundingBoxCoordinates.yMin + '-' + boundingBoxCoordinates.yMax } let pointCounterMax = 1000000 let limitConversionToFirstPoint = false // limitConversionToFirstPoint = true; if (limitConversionToFirstPoint) { - pointCounterMax = 1 + pointCounterMax = 1 } - -// let sourceFileChargemapJson = './chargemap_data/hurepoix.json' -let sourceFileIRVEGeoJson = './etalab_data/latest.json' let defaultPropertiesOfPoint = { - 'amenity': 'charging_station' + 'amenity': 'charging_station' } let listOfBooleanKeys = Object.keys({ - prise_type_ef: 'socket:typee', - prise_type_2: 'socket:type2', - prise_type_combo_ccs: 'socket:type2_combo', - prise_type_chademo: 'socket:chademo', - gratuit: 'fee', - paiement_acte: 'authentication:none', - paiement_cb: 'payment:credit_cards', - cable_t2_attache: 'socket:type2_cable', + prise_type_ef: 'socket:typee', + prise_type_2: 'socket:type2', + prise_type_combo_ccs: 'socket:type2_combo', + prise_type_chademo: 'socket:chademo', + gratuit: 'fee', + paiement_acte: 'authentication:none', + paiement_cb: 'payment:credit_cards', + cable_t2_attache: 'socket:type2_cable', }) -function debugLog (message) { - if (!show_debug) { - return - } - console.log('debug: ', ...message) + +let converted_geo_json = { + type: 'FeatureCollection', + features: [] +} +/** + * + * @param sourceFilePath + * @param mapping + * @param pointCounterMax + * @param boundingBoxCoordinates + */ +function convertDataForIRVE(sourceFilePath, mapping, pointCounterMax, boundingBoxCoordinates) { + debugLog('convertDataFromChargemap from ', sourceFilePath) + + + fs.readFile(sourceFilePath, 'utf8', function (err, data) { + let point_counter = 0 + if (err) { + return debugLog(err) + } + let data_transformed = JSON.parse(data) + // debug('data keys ', Object.keys(dataTransformed)) + + debugLog('debug: properties of point 0', data_transformed.features[0]) + if (data_transformed.features) { + + debugLog('data found, features:', data_transformed.features.length) + + // find interesting list of points to use + let list_of_points = data_transformed.features + // for each point from the data source, convert with the mapping + + console.log('listOfPoints.length', list_of_points.length) + list_of_points.forEach(feature_point => { + + let regex_filter_test_result = true + + if (enable_filter_on_department) { + console.log('filtre sur les départements activé') + regex_filter_test_result = ( + filterZipCode.test(feature_point.properties.consolidated_code_postal) + || + filterZipCodeAdresse.test(feature_point.properties.adresse_station) + ) + } else if (filterOnBoundingBox) { + console.log('filtre sur les coordonnées activé') + + let x = feature_point.properties.coordonneesXY[0] + let xMin = boundingBoxCoordinates.xMin + let xMax = boundingBoxCoordinates.xMax + let yMin = boundingBoxCoordinates.yMin + let yMax = boundingBoxCoordinates.yMax + + let y = feature_point.properties.coordonneesXY[1] + regex_filter_test_result = ( + (x >= xMin && x <= xMax) + && + (y >= yMin && y <= yMax) + ) + + } + + // filter points depending on zipcode + if (filterCoordinates && regex_filter_test_result) { + + debugLog('featurePoint.properties.consolidated_code_postal', feature_point.properties.consolidated_code_postal) + + // limit results number of points + // if (pointcounter < pointCounterMax) { + + debugLog('add point') + + debugLog('featurePoint', feature_point) + + let mapped_point = {}; + + if (use_mappping_engine) { + mapping_engine.setConfig(mapping) + mapped_point = mapping_engine.mapFeaturePoint(feature_point) + } else { + mapped_point = mapElementFromConfSimple(feature_point, mapping) + } + debugLog('map one point', feature_point, mapped_point) + if (mapped_point) { + converted_geo_json.features.push(mapped_point) + } + } + // } + point_counter++ + + }) + // output new geojson + console.log('convertedGeoJson.features.length', converted_geo_json.features.length) + // write file on disk + let fileNameToWrite = 'my_converted_data_set' + filteredName + '.json' + console.log('write file ', fileNameToWrite) + writeJsonFile(fileNameToWrite, JSON.stringify(converted_geo_json, null, 2)) + + debugLog('mapped output:', converted_geo_json.features) + + return converted_geo_json + } + }) } -function convertDataForIRVE (sourceFilePath, mapping, pointCounterMax, boundingBoxCoordinates) { - debugLog('convertDataFromChargemap from ', sourceFilePath) - - let convertedGeoJson = { - type: 'FeatureCollection', - features: [] - } - - fs.readFile(sourceFilePath, 'utf8', function (err, data) { - let pointcounter = 0 - if (err) { - return debugLog(err) - } - let dataTransformed = JSON.parse(data) - // debug('data keys ', Object.keys(dataTransformed)) - - debugLog('debug: properties of point 0', dataTransformed.features[0]) - if (dataTransformed.features) { - - debugLog('data found, features:', dataTransformed.features.length) - - // find interesting list of points to use - let listOfPoints = dataTransformed.features - // for each point from the data source, convert with the mapping - - console.log('listOfPoints.length', listOfPoints.length) - listOfPoints.forEach(featurePoint => { - - let regextestresult = true - - if (enableFilterOnDepartment) { - console.log('filtre sur les départements activé') - regextestresult = ( - filterZipCode.test(featurePoint.properties.consolidated_code_postal) - || - filterZipCodeAdresse.test(featurePoint.properties.adresse_station) - ) - } else if (filterOnBoundingBox) { - console.log('filtre sur les coordonnées activé') - - let x = featurePoint.properties.coordonneesXY[0] - let xMin = boundingBoxCoordinates.xMin - let xMax = boundingBoxCoordinates.xMax - let yMin = boundingBoxCoordinates.yMin - let yMax = boundingBoxCoordinates.yMax - - let y = featurePoint.properties.coordonneesXY[1] - regextestresult = ( - (x >= xMin && x <= xMax) - && - (y >= yMin && y <= yMax) - ) - - } - - // filter points depending on zipcode - if (filterCoordinates && regextestresult) { - - debugLog('featurePoint.properties.consolidated_code_postal', featurePoint.properties.consolidated_code_postal) - - // limit results number of points - // if (pointcounter < pointCounterMax) { - - debugLog('add point') - - debugLog('featurePoint', featurePoint) - - let mappedPoint = mapElementFromConf(featurePoint, mapping) - debugLog('map one point', featurePoint, mappedPoint) - if (mappedPoint) { - convertedGeoJson.features.push(mappedPoint) - } - } - // } - pointcounter++ - - }) - // output new geojson - console.log('convertedGeoJson.features.length', convertedGeoJson.features.length) - // write file on disk - let fileNameToWrite = 'my_converted_data_set' + filteredName + '.json' - console.log('write file ', fileNameToWrite) - writeJsonFile(fileNameToWrite, JSON.stringify(convertedGeoJson, null, 2)) - - debugLog('mapped output:', convertedGeoJson.features) - - return convertedGeoJson - } - }) -} - -function isBooleanKey (pointKeyName) { - - return listOfBooleanKeys.indexOf(pointKeyName) !== -1 -} /** * retuns the converted element from mapping config if present, null otherwise */ +function mapElementFromConfSimple(featurePoint, mappingConfig) +{ + let mappingKeys = Object.keys(mappingConfig) + let featurePointPropertiesKeys = Object.keys(featurePoint.properties) -function mapElementFromConf (featurePoint, mappingConfig) { - let mappingKeys = Object.keys(mappingConfig) - let featurePointPropertiesKeys = Object.keys(featurePoint.properties) + debugLog('keys', mappingKeys, featurePointPropertiesKeys) - debugLog('keys', mappingKeys, featurePointPropertiesKeys) + let newProperties = defaultPropertiesOfPoint - let newProperties = defaultPropertiesOfPoint + // reinit properties of current point + let basePoint = Object.create(featurePoint) + basePoint.type = featurePoint.type + basePoint.geometry = featurePoint.geometry + basePoint.properties = newProperties - // reinit properties of current point - let basePoint = Object.create(featurePoint) - basePoint.type = featurePoint.type - basePoint.geometry = featurePoint.geometry - basePoint.properties = newProperties + // apply new properties if found in mapping config + featurePointPropertiesKeys.forEach(pointKeyName => { - // apply new properties if found in mapping config - featurePointPropertiesKeys.forEach(pointKeyName => { + if (mappingKeys.indexOf(pointKeyName) !== -1) { + debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName]) + let convertedValue = '' + if (isBooleanKey(pointKeyName)) { - if (mappingKeys.indexOf(pointKeyName) !== -1) { - debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName]) - let convertedValue = '' - if (isBooleanKey(pointKeyName)) { + convertedValue = featurePoint.properties[pointKeyName].toLowerCase() == 'true' ? 'yes' : 'no' + } else { + convertedValue = featurePoint.properties[pointKeyName] + } - convertedValue = featurePoint.properties[pointKeyName].toLowerCase() == 'true' ? 'yes' : 'no' - } else { - convertedValue = featurePoint.properties[pointKeyName] - } + if (convertedValue) { + newProperties[mappingConfig[pointKeyName]] = convertedValue + } + } + }) - if (convertedValue) { - newProperties[mappingConfig[pointKeyName]] = convertedValue - } - } - }) - - debugLog('basePoint', basePoint) - return basePoint + debugLog('basePoint', basePoint) + return basePoint } -function writeJsonFile (fileName, fileContent) { - debugLog('write file ', fileName) +function writeJsonFile(fileName, fileContent) { + debugLog('write file ', fileName) - return fs.writeFile( - `./output/${fileName}`, - fileContent, - 'utf8', - (err) => { - if (err) { - debugLog(`Error writing file: ${err}`) - } else { - debugLog(`File ${fileName} is written successfully!`) - } - } - ) + return fs.writeFile( + `./output/${fileName}`, + fileContent, + 'utf8', + (err) => { + if (err) { + debugLog(`Error writing file: ${err}`) + } else { + debugLog(`File ${fileName} is written successfully!`) + } + } + ) } console.log('pointCounterMax', pointCounterMax) -convertDataForIRVE(sourceFileIRVEGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates) - -// convertDataFromChargemap(sourceFileChargemapJson, mappingConfigIRVE) +convertDataForIRVE(sourceFilePathGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates) diff --git a/mapping_geojson_to_osm_tags/dataset_info.md b/mapping_geojson_to_osm_tags/dataset_info.md index 44772866..8e78ef24 100644 --- a/mapping_geojson_to_osm_tags/dataset_info.md +++ b/mapping_geojson_to_osm_tags/dataset_info.md @@ -2,7 +2,7 @@ D'après le jeu de données Etalab -## liste des opérateurs +## liste des 310 opérateurs 270 AGENCY 2Ed Coutances 2F Production diff --git a/mapping_geojson_to_osm_tags/mappings/configIRVE.js b/mapping_geojson_to_osm_tags/mappings/configIRVE.js index 77f1cb31..6cf8c930 100644 --- a/mapping_geojson_to_osm_tags/mappings/configIRVE.js +++ b/mapping_geojson_to_osm_tags/mappings/configIRVE.js @@ -5,69 +5,82 @@ */ export default { - // ******* nombres - nbre_pdc: 'capacity', - // ******* textes - amenity: 'amenity', // conserver le tag de base - capacity: 'capacity', // conserver le tag de base - nom_amenageur: 'operator', - siren_amenageur: 'owner:ref:FR:SIREN', - nom_operateur: 'operator', - telephone_operateur: 'phone', - contact_operateur: 'email', // ici, on souhaite convertir la clé contact_operateur=bidule en email=bidule + config_name: "IRVE config", + config_author: "tykayn ", + default_properties_of_point: { + 'amenity': 'charging_station' + }, + tags: { - id_station_itinerance: 'ref:EU:EVSE', - id_station_local: 'ref', + // ******* nombres + nbre_pdc: 'capacity', + // ******* textes + amenity: 'amenity', // conserver le tag de base + capacity: 'capacity', // conserver le tag de base + nom_amenageur: 'operator', + siren_amenageur: 'owner:ref:FR:SIREN', + nom_operateur: 'operator', + telephone_operateur: 'phone', + contact_operateur: 'email', // ici, on souhaite convertir la clé contact_operateur=bidule en email=bidule - gratuit: 'fee', - paiement_acte: 'authentication:none', + id_station_itinerance: 'ref:EU:EVSE', + id_station_local: 'ref', - reservation: 'reservation', - observations: 'note', - nom_station: 'name', - nom_enseigne: 'network', + gratuit: 'fee', + paiement_acte: 'authentication:none', - // ******* dates - date_mise_en_service: 'start_date', - date_maj: 'source:date', + reservation: 'reservation', + observations: 'note', + nom_station: 'name', + nom_enseigne: 'network', - // TODO gestion des types dont on doit convertir la valeur - // ******** champs booléens - cable_t2_attache: 'socket:type2_cable', - prise_type_ef: 'socket:typee', - prise_type_2: 'socket:type2', - prise_type_combo_ccs: 'socket:type2_combo', - prise_type_chademo: 'socket:chademo', + // ******* dates + date_mise_en_service: 'start_date', + date_maj: 'source:date', - // ******** champs plus complexes - horaires: 'opening_hours', // déjà au bon format - - // accessibilite_pmr: 'wheelchair', - // paiement_cb: 'payment:credit_cards', - - accessibilite_pmr: { - key_converted: "wheelchair", - conditional_values: { - "Accessibilité inconnue": { - value_converted: "", - ignore_this_data: true, // ne pas ajouter de tag si la valeur est égale à Accessibilité inconnue. - // transform_function : (original_value) => original_value.toLowerCase(), - }, - "Accessible mais non réservé PMR": { - value_converted: "" - }, - "Réservé PMR": { - value_converted: "" - }, - "Non accessible": { - value_converted: "no" - }, - "Mo-Fr 08:30-12:00,Mo-Fr 14:00-19:00,Sat 09:00-18:30": { - value_converted: "Mo-Fr 08:30-12:00,Mo-Fr 14:00-19:00,Sat 09:00-18:30" - } - "24/7": { - value_converted: "" + // TODO gestion des types dont on doit convertir la valeur + // ******** champs booléens + cable_t2_attache: + { + key_converted: 'socket:type2_cable', + // cable_t2_attache + truthy_value: '1' } + , + prise_type_ef: 'socket:typee', + prise_type_2: 'socket:type2', + prise_type_combo_ccs: 'socket:type2_combo', + prise_type_chademo: 'socket:chademo', + + // ******** champs plus complexes + horaires: 'opening_hours', // déjà au bon format + + // accessibilite_pmr: 'wheelchair', + // paiement_cb: 'payment:credit_cards', + + accessibilite_pmr: { + key_converted: "wheelchair", + conditional_values: { + "Accessibilité inconnue": { + // value_converted: "", + ignore_this_data: true, // ne pas ajouter de tag si la valeur est égale à Accessibilité inconnue. + // transform_function : (original_value) => original_value.toLowerCase(), + }, + "Accessible mais non réservé PMR": { + value_converted: "yes" + }, + "Réservé PMR": { + value_converted: "yes" + }, + "Non accessible": { + value_converted: "no" + }, + "Mo-Fr 08:30-12:00,Mo-Fr 14:00-19:00,Sat 09:00-18:30": { + value_converted: "Mo-Fr 08:30-12:00,Mo-Fr 14:00-19:00,Sat 09:00-18:30" + }, + "24/7": { + value_converted: "" + } // choix: @@ -81,25 +94,20 @@ export default { // Mo-Fr 09:00-16:00 // Mo-Fr 08:00-12:00,Mo-Fr 14:00-18:00,Th 08:00-18:00 -} - }, - station_deux_roues: { - key_converted: null, - conditional_values: { - // ajout de trois tags si la valeur est yes - yes: { - tags_to_add: [ - {bicycle: "yes"}, - {scooter: "yes"}, - {motorcar: "no"}, - ] + } + }, + station_deux_roues: { + key_converted: null, + conditional_values: { + // ajout de trois tags si la valeur est yes + yes: { + tags_to_add: [ + {bicycle: "yes"}, + {scooter: "yes"}, + {motorcar: "no"}, + ] + } } } - - } - // station_deux_roues => - // bicycle=yes - // scooter=yes - // motorcar=no - + }, } \ No newline at end of file diff --git a/mapping_geojson_to_osm_tags/mappings/engine.js b/mapping_geojson_to_osm_tags/mappings/engine.js new file mode 100644 index 00000000..bc50b8e1 --- /dev/null +++ b/mapping_geojson_to_osm_tags/mappings/engine.js @@ -0,0 +1,75 @@ +import debugLog from 'utils' + +export default class { + config = {} + constructor(mappingConfig){ + + this.setConfig(mappingConfig) + } + setConfig(mappingConfig){ + this.config = mappingConfig + } + + mapFeaturePoint(featurePointGeoJson){ + + let geoJSONConvertedPoint = {} + geoJSONConvertedPoint.properties = {... this.config.default_properties_of_point}; + geoJSONConvertedPoint.type = featurePointGeoJson.type; + geoJSONConvertedPoint.geometry = featurePointGeoJson.geometry + + let props = featurePointGeoJson.properties + + props.forEach((key,value)=>{ + + }) + + return geoJSONConvertedPoint; + } + + + isBooleanKey (pointKeyName) { + + return listOfBooleanKeys.indexOf(pointKeyName) !== -1 + } + + /** + * retuns the converted element from mapping config if present, null otherwise + */ + mapElementFromConf (featurePoint, mappingConfig) { + let mappingKeys = Object.keys(mappingConfig) + let featurePointPropertiesKeys = Object.keys(featurePoint.properties) + + debugLog('keys', mappingKeys, featurePointPropertiesKeys) + + let newProperties = defaultPropertiesOfPoint + + // reinit properties of current point + let basePoint = Object.create(featurePoint) + basePoint.type = featurePoint.type + basePoint.geometry = featurePoint.geometry + basePoint.properties = newProperties + + // apply new properties if found in mapping config + featurePointPropertiesKeys.forEach(pointKeyName => { + + if (mappingKeys.indexOf(pointKeyName) !== -1) { + debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName]) + let convertedValue = '' + if (isBooleanKey(pointKeyName)) { + + convertedValue = featurePoint.properties[pointKeyName].toLowerCase() == 'true' ? 'yes' : 'no' + } else { + convertedValue = featurePoint.properties[pointKeyName] + } + + if (convertedValue) { + newProperties[mappingConfig[pointKeyName]] = convertedValue + } + } + }) + + debugLog('basePoint', basePoint) + return basePoint + } + +} \ No newline at end of file diff --git a/mapping_geojson_to_osm_tags/mappings/utils.js b/mapping_geojson_to_osm_tags/mappings/utils.js new file mode 100644 index 00000000..1ac71228 --- /dev/null +++ b/mapping_geojson_to_osm_tags/mappings/utils.js @@ -0,0 +1,18 @@ + + +let show_debug = 0 + +/** + * faire un log + * @param message + */ +function debugLog (message) { + if (!show_debug) { + return + } + console.log('debug: ', ...message) +} +export default +{ + debugLog +}