diff --git a/README.md b/README.md index b19ce82..aa5d562 100644 --- a/README.md +++ b/README.md @@ -16,10 +16,10 @@ pour le lancer: npm start. * avoir Nodejs en version stable et installer les modules node. # Utilisation -## Configuration +## Configuration Vérifiez les variables de chemin dans **parse_orgmode_to_json.mjs**, surtout outputAbsolutePath pour avoir un lieu de génération de fichier. -## compiler les fichiers org et les convertir en json +## compiler les fichiers org et les convertir en json Il suffit de lancer cette commande du Makefile ```bash make convert @@ -28,8 +28,8 @@ Elle réalise la concaténation des fichiers tasks.org et tasks.org_archive fait la sortie est générée dans "outputAbsolutePath" définie dans "parse_orgmode_to_json.mjs" -## Données générées -### Des statistiques +## Données générées +### Des statistiques * Les nombres et le type de tâches agrégées par semaine, mois et années. * le nombre de tâches avec une date. @@ -41,20 +41,17 @@ la sortie est générée dans "outputAbsolutePath" définie dans "parse_orgmode_ * Les nombres de fois que des mots sont utilisés dans les tâches. * des données sur l'auteur, la date de génération du json, le fichier source .org ayant servi à la conversion. - -### - -## voir le rendu +## voir le rendu Lancer l'exécution du fichier app avec node, et consulter l'output html. ```bash npm start ``` -## Memacs +## Memacs Les scripts inspirés de Memacs (le package python-pip) sont dans le dossier **converters**, ils permettent de convertir des fichiers exportés de divers outils numériques en feuilles de calcul pour avoir un historique général d'activités simple à lire pour les humains, et facile à analyser pour d'autres scripts. # feuille de route TODO -Proposer une sortie json qui soit du même format que l'export ox-json d'emacs. - +- Proposer une sortie json qui soit du même format que l'export ox-json d'emacs. +- Inclure les sous tâches aux catégories # DONE \ No newline at end of file diff --git a/parse_orgmode_to_json.mjs b/parse_orgmode_to_json.mjs index 284c83f..6102569 100644 --- a/parse_orgmode_to_json.mjs +++ b/parse_orgmode_to_json.mjs @@ -2,29 +2,28 @@ * convertir un fichier .org vers des données structurées en json * @type {*} */ -import fs from 'node-fs'; -import moment from 'moment'; -import * as emoji from "node-emoji"; - +import fs from 'node-fs' +import moment from 'moment' +import * as emoji from 'node-emoji' /********************** * initialize configs **********************/ const sourceFileName = 'all_tasks.org' -const sourceFilePath = './sources/' + sourceFileName; -const outputAbsolutePath = '~/Nextcloud/ressources/social sorting/output/'; -const outputFileNameJson = 'export_' + sourceFileName + '_parsed.json'; +const sourceFilePath = './sources/' + sourceFileName +const outputAbsolutePath = '~/Nextcloud/ressources/social sorting/output/' +const outputFileNameJson = 'export_' + sourceFileName + '_parsed.json' let headers = [] let tasksObjectsForJsonExport = [] let headersByKind = {} -let writeJsonAfterParse = false; -writeJsonAfterParse = true; +let writeJsonAfterParse = false +writeJsonAfterParse = true moment.locale('FR') -const tada = emoji.get("tada"); -const gift = emoji.get("gift"); +const tada = emoji.get('tada') +const gift = emoji.get('gift') /************************************************************** * fetch the source orgmode file to read its contents @@ -32,43 +31,43 @@ const gift = emoji.get("gift"); console.log('---------- parse some org file', sourceFilePath) if (!sourceFilePath) { - console.error('pas de fichier à ouvrir') + console.error('pas de fichier à ouvrir') } fs.stat(sourceFilePath, function (err, stat) { - if (err == null) { - console.log(`File ${sourceFilePath} exists`); + if (err === null) { + console.log(`File ${sourceFilePath} exists`) - } else if (err.code === 'ENOENT') { - // file does not exist - console.error(`le fichier ${sourceFilePath} est introuvable. Impossible d en extraire des infos.`, err); - } else { - console.log('Some other error: ', err.code); - } -}); + } else if (err.code === 'ENOENT') { + // file does not exist + console.error(`le fichier ${sourceFilePath} est introuvable. Impossible d en extraire des infos.`, err) + } else { + console.log('Some other error: ', err.code) + } +}) /********************** * search elements *********************/ -let stateKeywordList = ['SOMEDAY', 'NEXT', 'TODO', 'CANCELLED', 'DONE', 'WAITING']; -let dateKeywordList = ['CREATED', 'SCHEDULED', 'DEADLINE', 'CLOSED', 'Refiled']; -let sectionKeywordList = ['PROPERTIES', 'LOGBOOK', 'END']; +let stateKeywordList = ['SOMEDAY', 'NEXT', 'TODO', 'CANCELLED', 'DONE', 'WAITING'] +let dateKeywordList = ['CREATED', 'SCHEDULED', 'DEADLINE', 'CLOSED', 'Refiled'] +let sectionKeywordList = ['PROPERTIES', 'LOGBOOK', 'END'] let propertiesSection = {} // TODO properties listing let logBookSection = {} // TODO logbook listing let statistics = { - tags: {}, - words: {}, - dates: { - havingDate: 0, - havingNoDate: 0, - oldEst: 0, - mostRecent: 0, - years: {}, - weeks: {}, - months: {}, - days: {} - } + tags: {}, + words: {}, + dates: { + havingDate: 0, + havingNoDate: 0, + oldEst: 0, + mostRecent: 0, + years: {}, + weeks: {}, + months: {}, + days: {} + } } let headerKeywordSearch = '[' + stateKeywordList.join('|') + ']' @@ -77,93 +76,93 @@ let headerKeywordSearch = '[' + stateKeywordList.join('|') + ']' * @type {{level: string, header: string, dates: {CREATED: string, DONE: string, REFILED: string}, state: string, content: string, properties: {}, tags: [], tagsInherited: []}} */ let task = { - header: "", - level: "", - corpus: "", - state: "", - tags: [], - children: [], // TODO list children tasks with a reference to the parent when level is superior to previous task - tagsInherited: [], // TODO inherit tags - dates: { - havingDate: 0, - havingNoDate: 0, - oldEst: 0, - mostRecent: 0, - years: {}, - weeks: {}, - months: {}, - days: {}, - }, - logbook: {}, - properties: {}, + header: '', + level: '', + corpus: '', + state: '', + tags: [], + children: [], // TODO list children tasks with a reference to the parent when level is superior to previous task + tagsInherited: [], // TODO inherit tags + dates: { + havingDate: 0, + havingNoDate: 0, + oldEst: 0, + mostRecent: 0, + years: {}, + weeks: {}, + months: {}, + days: {}, + }, + logbook: {}, + properties: {}, } -let isHeader = false; -let isProperty = false; -let isLogbook = false; -let isFirst = true; +let isHeader = false +let isProperty = false +let isLogbook = false +let isFirst = true // init first task object as empty clone -let currentTask = Object.create(task); +let currentTask = Object.create(task) /** * add to tasks to export and refresh current task */ -function addAndRefreshCurrentTask() { +function addAndRefreshCurrentTask () { - makeWordsStatistics(currentTask.header.trim()) - makeWordsStatistics(currentTask.corpus.trim()) + makeWordsStatistics(currentTask.header.trim()) + makeWordsStatistics(currentTask.corpus.trim()) - tasksObjectsForJsonExport.push(currentTask) - // réinitialisation de tâche pour remplir de nouveau - currentTask = Object.create(task); - currentTask.dates = {}; - currentTask.tags = []; + tasksObjectsForJsonExport.push(currentTask) + // réinitialisation de tâche pour remplir de nouveau + currentTask = Object.create(task) + currentTask.dates = {} + currentTask.tags = [] } -function makeWordsStatistics(sentence) { - let split = sentence.split(' '); - if (split && split.length) { +function makeWordsStatistics (sentence) { + let split = sentence.split(' ') + if (split && split.length) { - split.forEach(word => { - if (!statistics.words[word]) { - statistics.words[word] = 0 - } - statistics.words[word]++ - }) - } + split.forEach(word => { + if (!statistics.words[word]) { + statistics.words[word] = 0 + } + statistics.words[word]++ + }) + } } const dateStats = { - created: 0, - refiled: 0, - closed: 0, - cancelled: 0, - scheduled: 0, - deadline: 0, - tasks_done: [] + created: 0, + refiled: 0, + closed: 0, + cancelled: 0, + scheduled: 0, + deadline: 0, + tasks_done: [] } -function fillPeriodTime(periodStat, keyword){ +function fillPeriodTime (periodStat, keyword) { - if (keyword === 'CLOSED') { - periodStat.closed++; - } - if (keyword === 'CREATED') { - periodStat.created++; - } - if (keyword === 'Refiled') { - periodStat.refiled++; - } - if (keyword === 'CANCELLED') { - periodStat.cancelled++; - } - if (keyword === 'SCHEDULED') { - periodStat.scheduled++; - } - if (keyword === 'DEADLINE') { - periodStat.deadline++; - } + if (keyword === 'CLOSED') { + periodStat.closed++ + } + if (keyword === 'CREATED') { + periodStat.created++ + } + if (keyword === 'Refiled') { + periodStat.refiled++ + } + if (keyword === 'CANCELLED') { + periodStat.cancelled++ + } + if (keyword === 'SCHEDULED') { + periodStat.scheduled++ + } + if (keyword === 'DEADLINE') { + periodStat.deadline++ + } } /** @@ -171,60 +170,59 @@ function fillPeriodTime(periodStat, keyword){ * @param keyword * @param dateFoundElement */ -function statisticDateFill(keyword, dateFoundElement, header) { +function statisticDateFill (keyword, dateFoundElement, header) { - // décompte par années + // décompte par années - let convertedDate = new Date(dateFoundElement) - let yearOfDate = convertedDate.getFullYear() - let monthOfDate = yearOfDate + '-' + convertedDate.getMonth() - // add zeros - let convertedWeek = moment(convertedDate).week() < 10 ? '0' + moment(convertedDate).week() : moment(convertedDate).week() - let weekOfDate = yearOfDate + '-' + convertedWeek + let convertedDate = new Date(dateFoundElement) + let yearOfDate = convertedDate.getFullYear() + let monthOfDate = yearOfDate + '-' + convertedDate.getMonth() + // add zeros + let convertedWeek = moment(convertedDate).week() < 10 ? '0' + moment(convertedDate).week() : moment(convertedDate).week() + let weekOfDate = yearOfDate + '-' + convertedWeek - let convertedMonth = convertedDate.getMonth() < 10 ? '0' + convertedDate.getMonth() : convertedDate.getMonth() - let convertedDay = convertedDate.getDay() < 10 ? '0' + convertedDate.getDay() : convertedDate.getDay() - let dayOfDate = convertedDate.getFullYear() + '-' + convertedMonth + '-' + convertedDay + let convertedMonth = convertedDate.getMonth() < 10 ? '0' + convertedDate.getMonth() : convertedDate.getMonth() + let convertedDay = convertedDate.getDay() < 10 ? '0' + convertedDate.getDay() : convertedDate.getDay() + let dayOfDate = convertedDate.getFullYear() + '-' + convertedMonth + '-' + convertedDay - if (!statistics.dates.years[yearOfDate]) { - statistics.dates.years[yearOfDate] = Object.create(dateStats) - } - fillPeriodTime(statistics.dates.years[yearOfDate], keyword) + if (!statistics.dates.years[yearOfDate]) { + statistics.dates.years[yearOfDate] = Object.create(dateStats) + } + fillPeriodTime(statistics.dates.years[yearOfDate], keyword) - // par année-semaine - if (!statistics.dates.weeks[weekOfDate]) { - statistics.dates.weeks[weekOfDate] = Object.create(dateStats) - } - fillPeriodTime(statistics.dates.weeks[weekOfDate], keyword) - statistics.dates.weeks[weekOfDate].tasks_done.push(header) - // décompte par mois - if (!statistics.dates.months[monthOfDate]) { - statistics.dates.months[monthOfDate] = Object.create(dateStats) - } - fillPeriodTime(statistics.dates.months[monthOfDate], keyword) + // par année-semaine + if (!statistics.dates.weeks[weekOfDate]) { + statistics.dates.weeks[weekOfDate] = Object.create(dateStats) + } + fillPeriodTime(statistics.dates.weeks[weekOfDate], keyword) + statistics.dates.weeks[weekOfDate].tasks_done.push(header) + // décompte par mois + if (!statistics.dates.months[monthOfDate]) { + statistics.dates.months[monthOfDate] = Object.create(dateStats) + } + fillPeriodTime(statistics.dates.months[monthOfDate], keyword) - // décompte par jours + // décompte par jours - if (!statistics.dates.days[dayOfDate]) { - statistics.dates.days[dayOfDate] = Object.create(dateStats) - } - fillPeriodTime(statistics.dates.days[dayOfDate], keyword) - statistics.dates.days[dayOfDate].tasks_done.push(header) + if (!statistics.dates.days[dayOfDate]) { + statistics.dates.days[dayOfDate] = Object.create(dateStats) + } + fillPeriodTime(statistics.dates.days[dayOfDate], keyword) + statistics.dates.days[dayOfDate].tasks_done.push(header) } -function findOldestDate(currentDate) { - // trouver la plus ancienne date +function findOldestDate (currentDate) { + // trouver la plus ancienne date - - if (!statistics.dates.oldEst) { - statistics.dates.oldEst = currentDate; - } else { - var beginningTime = moment(statistics.dates.oldEst); - var endTime = moment(currentDate); - if (!beginningTime.isBefore(endTime)) { - statistics.dates.oldEst = currentDate; - } - } + if (!statistics.dates.oldEst) { + statistics.dates.oldEst = currentDate + } else { + var beginningTime = moment(statistics.dates.oldEst) + var endTime = moment(currentDate) + if (!beginningTime.isBefore(endTime)) { + statistics.dates.oldEst = currentDate + } + } } /********************** @@ -232,208 +230,199 @@ function findOldestDate(currentDate) { *********************/ fs.readFile(sourceFilePath, 'utf8', function (err, data) { + if (err) { + return console.log(err) + } + console.log(' parsing...') + // parcourir chaque ligne du fichier org + let everyline = data.split('\n') - if (err) { - return console.log(err); - } - console.log(" parsing...") - // parcourir chaque ligne du fichier org - let everyline = data.split('\n'); + // trouver les entêtes toutes les lignes qui commencent par * et espace. - // trouver les entêtes toutes les lignes qui commencent par * et espace. + everyline.forEach((line) => { - everyline.forEach((line) => { + // gérer la création d'objets définissant les tâches et leurs propriétés + if (line.match(/^\*+? /)) { + // add last task to export list + if (!isFirst) { + addAndRefreshCurrentTask() + } else { + isFirst = false + } - // gérer la création d'objets définissant les tâches et leurs propriétés - if (line.match(/^\*+? /)) { - // add last task to export list - if (!isFirst) { + isHeader = true + // compter les étoiles pour trouver le niveau du header + if (line.match(/\*/g)) { - addAndRefreshCurrentTask(); - } else { - isFirst = false; - } + let match = line.match(/\*/g) + currentTask.level = match.length + } - isHeader = true; - // compter les étoiles pour trouver le niveau du header - if (line.match(/\*/g)) { + // create a new task - let match = line.match(/\*/g); - currentTask.level = match.length - } + headers.push(cleanHeader(line)) + currentTask.header = cleanHeader(line) + stateKeywordList.forEach(keyword => { + let keywordIsFound = lineHasKeyword(line, keyword) - // create a new task + if (keywordIsFound) { + currentTask.state = keyword + } + }) + // trouver les tags + let tagsList = line.match(/\:([\w\_]*)\:/g) + if (tagsList) { + tagsList = tagsList[0] + let tagList = tagsList.split(':') + if (tagList.length) { - headers.push(cleanHeader(line)) - currentTask.header = cleanHeader(line); - stateKeywordList.forEach(keyword => { - let keywordIsFound = lineHasKeyword(line, keyword) + tagList.forEach(tag => { + if (tag.length > 1) { - if (keywordIsFound) { - currentTask.state = keyword - } - }) + if (!statistics.tags[tag]) { + statistics.tags[tag] = 0 + } + statistics.tags[tag]++ + currentTask.tags.push(tag) + } + }) + } + } - // trouver les tags - let tagsList = line.match(/\:([\w\_]*)\:/g) - if (tagsList) { - tagsList = tagsList[0]; - let tagList = tagsList.split(':'); - if (tagList.length) { + // ------------- fin des recherches dans la ligne de Header ------------- + } else { + isHeader = false + } + // examen des lignes de corps de tâche, ou de corps de section suite au header. + // classer les dates de création, cloture, et de logbook + let dateFound = searchDate(line) + if (dateFound) { + /** + * we have found a date in the current line + */ - tagList.forEach(tag => { - if (tag.length > 1) { + statistics.dates.havingDate += 1 - if (!statistics.tags[tag]) { - statistics.tags[tag] = 0 - } - statistics.tags[tag]++ + dateKeywordList.forEach(keyword => { + if (lineHasSubstring(line, keyword)) { + if (!currentTask.dates[keyword]) { + currentTask.dates[keyword] = '' + } - currentTask.tags.push(tag) - } - }) - } - } + let convertedDate = dateFound[0].substring(0, 10) + if (dateFound[0].length === 15) { + // sans heure: "2022-12-21 mer." + convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd') - // ------------- fin des recherches dans la ligne de Header ------------- - } else { - isHeader = false; - } - // examen des lignes de corps de tâche, ou de corps de section suite au header. - // classer les dates de création, cloture, et de logbook - let dateFound = searchDate(line) - if (dateFound) { - /** - * we have found a date in the current line - */ + } else if (dateFound[0].length === 21) { + // avec heure: "2022-11-01 mar. 00:44" + convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd HH:mm') + } + let formattedDate = moment(convertedDate).format() + // console.log('currentTask.header', currentTask.header) + statisticDateFill(keyword, convertedDate, currentTask.header) + findOldestDate(convertedDate) + currentTask.dates[keyword] = formattedDate - statistics.dates.havingDate += 1; + } else { + // console.log('keyword', keyword) + } + }) + } else { + statistics.dates.havingNoDate += 1 + if ( + line.indexOf(dateKeywordList) !== -1 && + line.indexOf(stateKeywordList) !== -1 && + line.indexOf(sectionKeywordList) !== -1 + ) { - dateKeywordList.forEach(keyword => { - if (lineHasSubstring(line, keyword)) { - if (!currentTask.dates[keyword]) { - currentTask.dates[keyword] = ''; - } + // ajouter le corps complet de la section après le header + if (line.length && !isHeader) { - let convertedDate = dateFound[0].substring(0, 10); + let cleanedLine = line.replace(/\s\s/g, ' ') + cleanedLine = line.replace(/ {2,}/g, ' ') - if (dateFound[0].length == 15) { - // sans heure: "2022-12-21 mer." - convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd') + cleanedLine = cleanedLine.trim() - } else if (dateFound[0].length == 21) { - // avec heure: "2022-11-01 mar. 00:44" - convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd HH:mm') - } - let formattedDate = moment(convertedDate).format() - console.log('currentTask.header', currentTask.header) - statisticDateFill(keyword, convertedDate, currentTask.header) - findOldestDate(convertedDate) - currentTask.dates[keyword] = formattedDate; - - - } else { - // console.log('keyword', keyword) - } - }) - } else { - statistics.dates.havingNoDate += 1; - if ( - line.indexOf(dateKeywordList) !== -1 && - line.indexOf(stateKeywordList) !== -1 && - line.indexOf(sectionKeywordList) !== -1 - ) { - - // ajouter le corps complet de la section après le header - if (line.length && !isHeader) { - - let cleanedLine = line.replace(/\s\s/g, ' '); - cleanedLine = line.replace(/ {2,}/g, ' ') - - cleanedLine = cleanedLine.trim() - - currentTask.corpus += `${cleanedLine} + currentTask.corpus += `${cleanedLine} ` - } - } - } - }) + } + } + } + }) + // ajouter la dernière tâche parsée + addAndRefreshCurrentTask() - // ajouter la dernière tâche parsée - addAndRefreshCurrentTask(); + console.log('tasks : ', tasksObjectsForJsonExport.length) + console.log(tada + ' parsing fini ' + tada) - console.log(tada+ " parsing fini "+ tada) + // ranger par valeur décroissante les tags - // ranger par valeur décroissante les tags + let sorted_stats = [] + // rangement par valeur et par date + console.log('write file ', outputAbsolutePath, outputFileNameJson) + statistics.dates.years = sortByKey(statistics.dates.years) + statistics.dates.weeks = sortByKey(statistics.dates.weeks) + statistics.dates.months = sortByKey(statistics.dates.months) + statistics.dates.days = sortByKey(statistics.dates.days) - let sorted_stats = []; - // rangement par valeur et par date - console.log('write file ', outputAbsolutePath, outputFileNameJson); - statistics.dates.years = sortByKey(statistics.dates.years) - statistics.dates.weeks = sortByKey(statistics.dates.weeks) - statistics.dates.months = sortByKey(statistics.dates.months) - statistics.dates.days = sortByKey(statistics.dates.days) + statistics = sortByKey(statistics) - statistics = sortByKey(statistics) + const jsonContent = { + statistics: { + lines_count: everyline.length, + headers_count: headers.length, + statistics + }, + meta_data: { + author: '@tykayn@mastodon.Cipherbliss.com', + generated_at: new Date(), + generated_from_file: sourceFilePath + sourceFileName, + sources: 'https://forge.chapril.org/tykayn/org-report-stats.git' + }, + tasks_list: tasksObjectsForJsonExport + } + if (writeJsonAfterParse) { - const jsonContent = { - statistics: { - lines_count: everyline.length, - headers_count: headers.length, - statistics - }, - meta_data: { - author: '@tykayn@mastodon.Cipherbliss.com', - generated_at: new Date(), - generated_from_file: sourceFilePath + sourceFileName, - sources: 'https://forge.chapril.org/tykayn/org-report-stats.git' - }, - tasks_list: tasksObjectsForJsonExport - } - - - if (writeJsonAfterParse) { - - writeFileInOuputFolderFromJsonObject(outputFileNameJson, jsonContent); - } + writeFileInOuputFolderFromJsonObject(outputFileNameJson, jsonContent) + } }) +function lineHasKeyword (line, keyword = 'TODO') { -function lineHasKeyword(line, keyword = 'TODO') { - - let isFound = (line.indexOf('* ' + keyword) !== -1) - if (isFound) { - createNewHeaderKind(keyword) - headersByKind[keyword].push(line); - if (!statistics[keyword]) { - statistics[keyword] = 0 - } - statistics[keyword]++ - } - return isFound; + let isFound = (line.indexOf('* ' + keyword) !== -1) + if (isFound) { + createNewHeaderKind(keyword) + headersByKind[keyword].push(line) + if (!statistics[keyword]) { + statistics[keyword] = 0 + } + statistics[keyword]++ + } + return isFound } -function lineHasSubstring(line, keyword) { - let isFound = (line.indexOf(keyword) !== -1) - if (!statistics[keyword]) { - statistics[keyword] = 0 - } - statistics[keyword]++ +function lineHasSubstring (line, keyword) { + let isFound = (line.indexOf(keyword) !== -1) + if (!statistics[keyword]) { + statistics[keyword] = 0 + } + statistics[keyword]++ - return isFound + return isFound } -function createNewHeaderKind(keyword) { - if (!headersByKind[keyword]) { - headersByKind[keyword] = []; - } +function createNewHeaderKind (keyword) { + if (!headersByKind[keyword]) { + headersByKind[keyword] = [] + } } /** @@ -443,22 +432,22 @@ function createNewHeaderKind(keyword) { * @param line * @returns {*} */ -function searchDate(line) { - // return line.match(/[(\d{4}\-\d{2}\-\d{2} ?\d{2}?\:?\d{2}?\:?\d{2}?)(\d{4}\-\d{2}\-\d{2})]/) - let simpleDay = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.?/) - let simpleDayHour = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.? \d{2}\:\d{2}/) - let simpleDayHourSec = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.? \d{2}\:\d{2}\:\d{2}/) +function searchDate (line) { + // return line.match(/[(\d{4}\-\d{2}\-\d{2} ?\d{2}?\:?\d{2}?\:?\d{2}?)(\d{4}\-\d{2}\-\d{2})]/) + let simpleDay = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.?/) + let simpleDayHour = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.? \d{2}\:\d{2}/) + let simpleDayHourSec = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.? \d{2}\:\d{2}\:\d{2}/) - if (simpleDayHourSec) { - return simpleDayHourSec; - } + if (simpleDayHourSec) { + return simpleDayHourSec + } - if (simpleDayHour) { - return simpleDayHour; - } - if (simpleDay) { - return simpleDay; - } + if (simpleDayHour) { + return simpleDayHour + } + if (simpleDay) { + return simpleDay + } } @@ -466,17 +455,17 @@ function searchDate(line) { * get the cleaned content of the header * @param line */ -function cleanHeader(line) { +function cleanHeader (line) { - line = '' + line; - stateKeywordList.forEach(keyword => { - line = line.replace(keyword, '') - }) - line = line.replace(/\** /, ''); - line = line.replace(/\[.*\]/g, ''); - line = line.replace(/\:.*\:/g, ''); - line = line.replace(' ', ''); - return line.trim(); + line = '' + line + stateKeywordList.forEach(keyword => { + line = line.replace(keyword, '') + }) + line = line.replace(/\** /, '') + line = line.replace(/\[.*\]/g, '') + line = line.replace(/\:.*\:/g, '') + line = line.replace(' ', '') + return line.trim() } /** @@ -484,14 +473,14 @@ function cleanHeader(line) { * @param objectStuff * @returns {{}} */ -function sortByKey(objectStuff) { - return Object.keys(objectStuff).sort().reduce( - (obj, key) => { - obj[key] = objectStuff[key]; - return obj; - }, - {} - ); +function sortByKey (objectStuff) { + return Object.keys(objectStuff).sort().reduce( + (obj, key) => { + obj[key] = objectStuff[key] + return obj + }, + {} + ) } /** @@ -499,32 +488,32 @@ function sortByKey(objectStuff) { * @param literalobject * @returns {any} */ -function sortByValue(literalobject) { +function sortByValue (literalobject) { - let sortable = []; - for (var keyName in literalobject) { - sortable[keyName] = literalobject[keyName]; - } - // return literalobject - return sortable.sort(function (a, b) { - return b[1] - a[1]; - }); + let sortable = [] + for (var keyName in literalobject) { + sortable[keyName] = literalobject[keyName] + } + // return literalobject + return sortable.sort(function (a, b) { + return b[1] - a[1] + }) } -export async function writeFileInOuputFolderFromJsonObject(fileName, jsonObjectThing) { +export async function writeFileInOuputFolderFromJsonObject (fileName, jsonObjectThing) { - // console.log('statistics.dates', statistics.dates) + // console.log('statistics.dates', statistics.dates) - return await fs.writeFile( - `${outputAbsolutePath}${fileName}`, - JSON.stringify(jsonObjectThing, null,2), - "utf8", - (err) => { - if (err) { - console.log(`Error writing file: ${err}`); - } else { - console.log(`\n ${gift} File ${fileName} is written successfully!`); - } - } - ); + return await fs.writeFile( + `${outputAbsolutePath}${fileName}`, + JSON.stringify(jsonObjectThing, null, 2), + 'utf8', + (err) => { + if (err) { + console.log(`Error writing file: ${err}`) + } else { + console.log(`\n ${gift} File ${fileName} is written successfully!`) + } + } + ) } \ No newline at end of file diff --git a/sms_to_csv.mjs b/sms_to_csv.mjs index 7d8e4e5..afcc60f 100644 --- a/sms_to_csv.mjs +++ b/sms_to_csv.mjs @@ -12,11 +12,12 @@ import { writeFileInOuputFolderFromJsonObject } from './utils.mjs' **********************/ // const sourceFileBaseName = 'sms-20180423162531' -const sourceFileBaseName = 'sms' +const sourceFileBaseName = 'sms-20230921224605' const sourceFileName = `${sourceFileBaseName}.xml` const outputFileJson = `${sourceFileBaseName}.json` -const outputFileCsv = `${sourceFileBaseName}.tsv` +const outputFileCsv = `${sourceFileBaseName}.csv` const folder_base = '/home/tykayn/Nextcloud/ressources/social sorting/' +const csv_separator = ';' const outputFileJsonPathFull = `${folder_base}output/${outputFileJson}` const sourceFileJson = `${folder_base}output/${outputFileJson}` @@ -54,46 +55,49 @@ function convertJsonToCsvAndPersistTSVFile (jsonFilePath, outputFileTSVName) { console.log('convert', folder_base+'output/' + jsonFilePath) fs.readFile(folder_base+'output/' + jsonFilePath, 'utf8', function (err, data) { - + data = JSON.parse(data) console.log('data', data.length) if (data) { const events = [ - 'amount\t' + - 'content\t' + - 'description\t' + - 'destination\t' + - 'end\t' + - 'kind of activity\t' + - 'person\t' + - 'place\t' + - 'source\t' + - 'start\t' + - 'unique id\t' + - 'url\t' + 'amount' + csv_separator + ''+ + 'content' + csv_separator +''+ + 'description' + csv_separator + + 'destination' + csv_separator + + 'end' + csv_separator + + 'kind of activity' + csv_separator + + 'person' + csv_separator + + 'place' + csv_separator + + 'source' + csv_separator + + 'start' + csv_separator + + 'unique id' + csv_separator + + 'url' + csv_separator ] - data = JSON.parse(data) - // console.log('data.length', data.length) - console.log('data', Object.keys(data)) - let smslist = data['elements'][1]['elements'] + + // console.log('data', data) + // console.log('data other ', data) + console.log('data.length', data['elements'].length) + console.log('data name', data['elements'][2]['elements'].length) + let smslist = data['elements'][2]['elements'] + // let smslist = data['smses'] if (smslist) { smslist.forEach(item => { // convert all fields to common event description events.push( - '\t' + - item.attributes.body.replace('\n', ' ') + '\t' + - 'sms ' + item.attributes.address + ' le ' + item.attributes.readable_date + '\t' + - item.attributes.address + '\t' + - '' + '\t' + - '' + '\t' + - item.attributes.contact_name + '\t' + - '' + '\t' + - '' + '\t' + - '' + '\t' + - '' + '\t' + - '' + '\t' + ''+csv_separator + + item.attributes.body + csv_separator + + 'sms ' + item.attributes.address + ' le ' + item.attributes.readable_date + csv_separator + + item.attributes.address + csv_separator + + '' + csv_separator + + '' + csv_separator + + item.attributes.contact_name + csv_separator + + '' + csv_separator + + '' + csv_separator + + '' + csv_separator + + '' + csv_separator + + '' + csv_separator ) }) // // console.log('events', events) @@ -122,10 +126,11 @@ function openSourceFile () { fs.readFile(sourceFilePath, 'utf8', function (err, data) { // // console.log('data', data) var jsonConversion = convert.xml2json(data) - console.log('jsonConversion keys', Object.keys(jsonConversion)) + // console.log('jsonConversion 0', Object.keys(jsonConversion['elements'])) // console.log('jsonConversion.length', parseFloat(jsonConversion.length / 1024 / 1024, 2), 'Mo') - writeFileInOuputFolderFromJsonObject(outputFileJson, JSON.parse( jsonConversion)) + // writeFileInOuputFolderFromJsonObject(outputFileJson, JSON.parse( jsonConversion)) + writeFileInOuputFolderFromJsonObject(outputFileJson, JSON.parse(jsonConversion)) convertJsonToCsvAndPersistTSVFile(outputFileJson, outputFileCsv) }) diff --git a/structure.json b/structure.json new file mode 100644 index 0000000..e69de29 diff --git a/utils.mjs b/utils.mjs index 4ea41ac..2fa85b8 100644 --- a/utils.mjs +++ b/utils.mjs @@ -20,11 +20,11 @@ export async function writeFileInOuputFolderFromJsonObject (fileName, jsonObject // console.log('statistics.dates', statistics.dates) let fullpath = `${outputAbsolutePath}${fileName}` - return await fs.writeFile( + return await fs.writeFileSync( fullpath, - JSON.stringify(jsonObjectThing, null, 2), - // JSON.stringify(jsonObjectThing).replace('\n',''), - 'utf8', + // JSON.stringify(jsonObjectThing, null, 2), + JSON.stringify(jsonObjectThing).replace('\n',''), + 'UTF8', (err) => { if (err) { console.log(`Error writing file: ${err}`)