From 177744ca6127bfc7167d9bae16395453bef7d377 Mon Sep 17 00:00:00 2001 From: Tykayn Date: Mon, 31 Jul 2023 19:27:17 +0200 Subject: [PATCH] engine runs --- .../convert_to_osm_tags.mjs | 42 +++++++--------- .../mappings/engine.mjs | 49 +++++++++++++------ .../mappings/infer_domain_from_csv.mjs | 29 ++++++----- .../mappings/utils.mjs | 25 ++++++++-- 4 files changed, 86 insertions(+), 59 deletions(-) diff --git a/mapping_geojson_to_osm_tags/convert_to_osm_tags.mjs b/mapping_geojson_to_osm_tags/convert_to_osm_tags.mjs index bfff7b3f..875d7f0a 100644 --- a/mapping_geojson_to_osm_tags/convert_to_osm_tags.mjs +++ b/mapping_geojson_to_osm_tags/convert_to_osm_tags.mjs @@ -10,10 +10,13 @@ import custom_utils from './mappings/utils.mjs' const { debugLog } = custom_utils const { isBooleanKey } = custom_utils +const { writeFile } = custom_utils // let debugLog = custom_utils.debugLog let use_mappping_engine = false +use_mappping_engine = true + let Mapping_engine = new mapping_engine(mappingConfigIRVE) let mini_arguments = minimist(process.argv.slice(2)) @@ -34,6 +37,7 @@ let boundingBoxCoordinates = { let filterCoordinates = true let enable_filter_on_department = true +enable_filter_on_department = false let filterDepartment = 91 if (mini_arguments['department']) { filterDepartment = mini_arguments['department'] @@ -56,7 +60,7 @@ let pointCounterMax = 1000000 let limitConversionToFirstPoint = false limitConversionToFirstPoint = true if (limitConversionToFirstPoint) { - pointCounterMax = 1 + pointCounterMax = 2 } let defaultPropertiesOfPoint = { 'amenity': 'charging_station' @@ -139,8 +143,9 @@ function convertDataForIRVE (sourceFilePath, mapping, pointCounterMax, boundingB let mapped_point = {} if (use_mappping_engine) { + console.log('go mapping engine') Mapping_engine.setConfig(mapping) - mapped_point = Mapping_engine.mapFeaturePoint(feature_point) + mapped_point = Mapping_engine.mapElementFromConf(feature_point) } else { mapped_point = mapElementFromConfSimple(feature_point, mapping) } @@ -158,7 +163,7 @@ function convertDataForIRVE (sourceFilePath, mapping, pointCounterMax, boundingB // write file on disk let fileNameToWrite = 'my_converted_data_set' + filteredName + '.json' console.log('write file ', fileNameToWrite) - writeJsonFile(fileNameToWrite, JSON.stringify(converted_geo_json, null, 2)) + writeFile(fileNameToWrite, JSON.stringify(converted_geo_json, null, 2)) debugLog('mapped output:', converted_geo_json.features) @@ -211,29 +216,16 @@ function mapElementFromConfSimple (featurePoint, mappingConfig) { return basePoint } -function writeJsonFile (fileName, fileContent) { - debugLog('write file ', fileName) - - return fs.writeFile( - `./output/${fileName}`, - fileContent, - 'utf8', - (err) => { - if (err) { - debugLog(`Error writing file: ${err}`) - } else { - debugLog(`File ${fileName} is written successfully!`) - } - } - ) -} - -console.log('pointCounterMax', pointCounterMax) - -if (!use_mappping_engine) { - convertDataForIRVE(sourceFilePathGeoJson, mappingConfigIRVE_simple, pointCounterMax, boundingBoxCoordinates) -} else { +if (use_mappping_engine) { + console.log('using mapping engine') + console.log('pointCounterMax', pointCounterMax) convertDataForIRVE(sourceFilePathGeoJson, mappingConfigIRVE, pointCounterMax, boundingBoxCoordinates) + +} else { + + convertDataForIRVE(sourceFilePathGeoJson, mappingConfigIRVE_simple, pointCounterMax, boundingBoxCoordinates) } + + diff --git a/mapping_geojson_to_osm_tags/mappings/engine.mjs b/mapping_geojson_to_osm_tags/mappings/engine.mjs index 20eddb1a..00fef4f1 100644 --- a/mapping_geojson_to_osm_tags/mappings/engine.mjs +++ b/mapping_geojson_to_osm_tags/mappings/engine.mjs @@ -1,7 +1,23 @@ -import debugLog from './utils.mjs' + +import custom_utils from './utils.mjs' + + +const { debugLog } = custom_utils + +let listOfBooleanKeys = Object.keys({ + prise_type_ef: 'socket:typee', + prise_type_2: 'socket:type2', + prise_type_combo_ccs: 'socket:type2_combo', + prise_type_chademo: 'socket:chademo', + gratuit: 'fee', + paiement_acte: 'authentication:none', + paiement_cb: 'payment:credit_cards', + cable_t2_attache: 'socket:type2_cable', +}) + export default class { - config = {} + mapping_config = {} constructor (mappingConfig) { @@ -9,13 +25,13 @@ export default class { } setConfig (mappingConfig) { - this.config = mappingConfig + this.mapping_config = mappingConfig } mapFeaturePoint (featurePointGeoJson) { let geoJSONConvertedPoint = {} - geoJSONConvertedPoint.properties = { ...this.config.default_properties_of_point } + geoJSONConvertedPoint.properties = { ...this.mapping_config.default_properties_of_point } geoJSONConvertedPoint.type = featurePointGeoJson.type geoJSONConvertedPoint.geometry = featurePointGeoJson.geometry @@ -39,13 +55,14 @@ export default class { /** * retuns the converted element from mapping config if present, null otherwise */ - mapElementFromConf (featurePoint, mappingConfig) { - let mappingKeys = Object.keys(mappingConfig) + mapElementFromConf (featurePoint) { + console.log('config_name', this.mapping_config.config_name) + let mappingKeys = Object.keys(this.mapping_config.tags) let featurePointPropertiesKeys = Object.keys(featurePoint.properties) - debugLog('keys', mappingKeys, featurePointPropertiesKeys) + console.log('keys', mappingKeys, featurePointPropertiesKeys) - let newProperties = defaultPropertiesOfPoint + let newProperties = Object.create(this.mapping_config.default_properties_of_point) // reinit properties of current point let basePoint = Object.create(featurePoint) @@ -56,8 +73,8 @@ export default class { // apply new properties if found in mapping config featurePointPropertiesKeys.forEach(pointKeyName => { - if (mappingKeys.indexOf(pointKeyName) !== -1) { - debugLog('found element', pointKeyName, '=>', mappingConfig[pointKeyName], 'value : ', featurePoint.properties[pointKeyName]) + if (!mappingKeys.indexOf(pointKeyName) !== -1) { + debugLog('found element', pointKeyName, '=>', this.mapping_config[pointKeyName], 'value : ', featurePoint.properties[pointKeyName]) let convertedValue = '' let valueConvertedFromMapping = featurePoint.properties[pointKeyName] @@ -68,14 +85,16 @@ export default class { let isConfigMappingObject = typeofValue === 'string' if (isStringValue) { - if (isBooleanKey(pointKeyName)) { - convertedValue = (valueConvertedFromMapping + '').toLowerCase() == 'true' ? 'yes' : 'no' + console.log('string value') + if (this.isBooleanKey(pointKeyName)) { + let lowerValue = (valueConvertedFromMapping + '').toLowerCase() + convertedValue = this.truthyValues.indexOf(lowerValue) ? 'yes' : 'no' } else { convertedValue = valueConvertedFromMapping } if (convertedValue) { - newProperties[mappingConfig[pointKeyName]] = convertedValue + newProperties[this.mapping_config[pointKeyName]] = convertedValue } } else if (isConfigMappingObject) { let newKey = '' @@ -103,12 +122,12 @@ export default class { // convertir la valeur, si elle est truthy, la transformer en ce que donne la propriété truthy_value // exemple: le jeu de données dit que la colonne cable_t2_attache vaut "True", mais on veut le convertir en "1". // on met donc truthy_value: '1' - if (this.truthyValues.indexOf( valueConvertedFromMapping) !== -1) { + if (this.truthyValues.indexOf(valueConvertedFromMapping) !== -1) { convertedValue = conditionnalConfig.truthy_value } } if (conditionnalConfig.falsy_value) { - if (this.falsyValues.indexOf( valueConvertedFromMapping) !== -1) { + if (this.falsyValues.indexOf(valueConvertedFromMapping) !== -1) { convertedValue = conditionnalConfig.falsy_value } } diff --git a/mapping_geojson_to_osm_tags/mappings/infer_domain_from_csv.mjs b/mapping_geojson_to_osm_tags/mappings/infer_domain_from_csv.mjs index 38e591d3..ab51bb95 100644 --- a/mapping_geojson_to_osm_tags/mappings/infer_domain_from_csv.mjs +++ b/mapping_geojson_to_osm_tags/mappings/infer_domain_from_csv.mjs @@ -1,5 +1,3 @@ - - let show_debug = 0 /** @@ -7,26 +5,27 @@ let show_debug = 0 * @param message */ function debugLog (message) { - if (!show_debug) { - return - } - console.log('debug: ', ...message) + if (!show_debug) { + return + } + console.log('debug: ', ...message) } - /** - * + * TODO + * find the domain of all columns in a csv, + * take the header and the values, return only the unique values for each column, save a new csv, + and build a config for the mapping engine listing all the possible values. * @param pointKeyName * @returns {boolean} */ -function openCSV(filePath) { +function openCSV (filePath) { - console.log('open csv filePath') - return + console.log('open csv filePath') + return } -export default -{ - debugLog, - isBooleanKey +export default { + debugLog, + isBooleanKey } diff --git a/mapping_geojson_to_osm_tags/mappings/utils.mjs b/mapping_geojson_to_osm_tags/mappings/utils.mjs index f1958744..ca549687 100644 --- a/mapping_geojson_to_osm_tags/mappings/utils.mjs +++ b/mapping_geojson_to_osm_tags/mappings/utils.mjs @@ -1,6 +1,6 @@ +import fs from 'node-fs' - -let show_debug = 0 +let show_debug = 1 /** * faire un log @@ -10,7 +10,7 @@ function debugLog (message) { if (!show_debug) { return } - console.log('debug: ', ...message) + console.log('debug: ', message) } @@ -37,8 +37,25 @@ function isBooleanKey(pointKeyName) { return listOfBooleanKeys.indexOf(pointKeyName) !== -1 } +function writeFile (fileName, fileContent) { + debugLog('write file ', fileName) + + return fs.writeFile( + `./output/${fileName}`, + fileContent, + 'utf8', + (err) => { + if (err) { + debugLog(`Error writing file: ${err}`) + } else { + debugLog(`File ${fileName} is written successfully!`) + } + } + ) +} export default { debugLog, - isBooleanKey + isBooleanKey, + writeFile }