hop
This commit is contained in:
Tykayn
tykayn
parent
37fc569fa3
commit
8c603e291e
19
README.md
19
README.md
|
|
@ -16,10 +16,10 @@ pour le lancer: npm start.
|
|||
* avoir Nodejs en version stable et installer les modules node.
|
||||
|
||||
# Utilisation
|
||||
## Configuration
|
||||
## Configuration
|
||||
Vérifiez les variables de chemin dans **parse_orgmode_to_json.mjs**, surtout outputAbsolutePath pour avoir un lieu de génération de fichier.
|
||||
|
||||
## compiler les fichiers org et les convertir en json
|
||||
## compiler les fichiers org et les convertir en json
|
||||
Il suffit de lancer cette commande du Makefile
|
||||
```bash
|
||||
make convert
|
||||
|
|
@ -28,8 +28,8 @@ Elle réalise la concaténation des fichiers tasks.org et tasks.org_archive fait
|
|||
|
||||
la sortie est générée dans "outputAbsolutePath" définie dans "parse_orgmode_to_json.mjs"
|
||||
|
||||
## Données générées
|
||||
### Des statistiques
|
||||
## Données générées
|
||||
### Des statistiques
|
||||
|
||||
* Les nombres et le type de tâches agrégées par semaine, mois et années.
|
||||
* le nombre de tâches avec une date.
|
||||
|
|
@ -41,20 +41,17 @@ la sortie est générée dans "outputAbsolutePath" définie dans "parse_orgmode_
|
|||
* Les nombres de fois que des mots sont utilisés dans les tâches.
|
||||
* des données sur l'auteur, la date de génération du json, le fichier source .org ayant servi à la conversion.
|
||||
|
||||
|
||||
###
|
||||
|
||||
## voir le rendu
|
||||
## voir le rendu
|
||||
Lancer l'exécution du fichier app avec node, et consulter l'output html.
|
||||
```bash
|
||||
npm start
|
||||
```
|
||||
|
||||
## Memacs
|
||||
## Memacs
|
||||
Les scripts inspirés de Memacs (le package python-pip) sont dans le dossier **converters**, ils permettent de convertir des fichiers exportés de divers outils numériques en feuilles de calcul pour avoir un historique général d'activités simple à lire pour les humains, et facile à analyser pour d'autres scripts.
|
||||
|
||||
|
||||
# feuille de route TODO
|
||||
Proposer une sortie json qui soit du même format que l'export ox-json d'emacs.
|
||||
|
||||
- Proposer une sortie json qui soit du même format que l'export ox-json d'emacs.
|
||||
- Inclure les sous tâches aux catégories
|
||||
# DONE
|
||||
|
|
@ -2,29 +2,28 @@
|
|||
* convertir un fichier .org vers des données structurées en json
|
||||
* @type {*}
|
||||
*/
|
||||
import fs from 'node-fs';
|
||||
import moment from 'moment';
|
||||
import * as emoji from "node-emoji";
|
||||
|
||||
import fs from 'node-fs'
|
||||
import moment from 'moment'
|
||||
import * as emoji from 'node-emoji'
|
||||
|
||||
/**********************
|
||||
* initialize configs
|
||||
**********************/
|
||||
|
||||
const sourceFileName = 'all_tasks.org'
|
||||
const sourceFilePath = './sources/' + sourceFileName;
|
||||
const outputAbsolutePath = '~/Nextcloud/ressources/social sorting/output/';
|
||||
const outputFileNameJson = 'export_' + sourceFileName + '_parsed.json';
|
||||
const sourceFilePath = './sources/' + sourceFileName
|
||||
const outputAbsolutePath = '~/Nextcloud/ressources/social sorting/output/'
|
||||
const outputFileNameJson = 'export_' + sourceFileName + '_parsed.json'
|
||||
|
||||
let headers = []
|
||||
let tasksObjectsForJsonExport = []
|
||||
let headersByKind = {}
|
||||
let writeJsonAfterParse = false;
|
||||
writeJsonAfterParse = true;
|
||||
let writeJsonAfterParse = false
|
||||
writeJsonAfterParse = true
|
||||
moment.locale('FR')
|
||||
|
||||
const tada = emoji.get("tada");
|
||||
const gift = emoji.get("gift");
|
||||
const tada = emoji.get('tada')
|
||||
const gift = emoji.get('gift')
|
||||
|
||||
/**************************************************************
|
||||
* fetch the source orgmode file to read its contents
|
||||
|
|
@ -32,43 +31,43 @@ const gift = emoji.get("gift");
|
|||
|
||||
console.log('---------- parse some org file', sourceFilePath)
|
||||
if (!sourceFilePath) {
|
||||
console.error('pas de fichier à ouvrir')
|
||||
console.error('pas de fichier à ouvrir')
|
||||
}
|
||||
fs.stat(sourceFilePath, function (err, stat) {
|
||||
if (err == null) {
|
||||
console.log(`File ${sourceFilePath} exists`);
|
||||
if (err === null) {
|
||||
console.log(`File ${sourceFilePath} exists`)
|
||||
|
||||
} else if (err.code === 'ENOENT') {
|
||||
// file does not exist
|
||||
console.error(`le fichier ${sourceFilePath} est introuvable. Impossible d en extraire des infos.`, err);
|
||||
} else {
|
||||
console.log('Some other error: ', err.code);
|
||||
}
|
||||
});
|
||||
} else if (err.code === 'ENOENT') {
|
||||
// file does not exist
|
||||
console.error(`le fichier ${sourceFilePath} est introuvable. Impossible d en extraire des infos.`, err)
|
||||
} else {
|
||||
console.log('Some other error: ', err.code)
|
||||
}
|
||||
})
|
||||
|
||||
/**********************
|
||||
* search elements
|
||||
*********************/
|
||||
let stateKeywordList = ['SOMEDAY', 'NEXT', 'TODO', 'CANCELLED', 'DONE', 'WAITING'];
|
||||
let dateKeywordList = ['CREATED', 'SCHEDULED', 'DEADLINE', 'CLOSED', 'Refiled'];
|
||||
let sectionKeywordList = ['PROPERTIES', 'LOGBOOK', 'END'];
|
||||
let stateKeywordList = ['SOMEDAY', 'NEXT', 'TODO', 'CANCELLED', 'DONE', 'WAITING']
|
||||
let dateKeywordList = ['CREATED', 'SCHEDULED', 'DEADLINE', 'CLOSED', 'Refiled']
|
||||
let sectionKeywordList = ['PROPERTIES', 'LOGBOOK', 'END']
|
||||
|
||||
let propertiesSection = {} // TODO properties listing
|
||||
let logBookSection = {} // TODO logbook listing
|
||||
|
||||
let statistics = {
|
||||
tags: {},
|
||||
words: {},
|
||||
dates: {
|
||||
havingDate: 0,
|
||||
havingNoDate: 0,
|
||||
oldEst: 0,
|
||||
mostRecent: 0,
|
||||
years: {},
|
||||
weeks: {},
|
||||
months: {},
|
||||
days: {}
|
||||
}
|
||||
tags: {},
|
||||
words: {},
|
||||
dates: {
|
||||
havingDate: 0,
|
||||
havingNoDate: 0,
|
||||
oldEst: 0,
|
||||
mostRecent: 0,
|
||||
years: {},
|
||||
weeks: {},
|
||||
months: {},
|
||||
days: {}
|
||||
}
|
||||
}
|
||||
|
||||
let headerKeywordSearch = '[' + stateKeywordList.join('|') + ']'
|
||||
|
|
@ -77,93 +76,93 @@ let headerKeywordSearch = '[' + stateKeywordList.join('|') + ']'
|
|||
* @type {{level: string, header: string, dates: {CREATED: string, DONE: string, REFILED: string}, state: string, content: string, properties: {}, tags: [], tagsInherited: []}}
|
||||
*/
|
||||
let task = {
|
||||
header: "",
|
||||
level: "",
|
||||
corpus: "",
|
||||
state: "",
|
||||
tags: [],
|
||||
children: [], // TODO list children tasks with a reference to the parent when level is superior to previous task
|
||||
tagsInherited: [], // TODO inherit tags
|
||||
dates: {
|
||||
havingDate: 0,
|
||||
havingNoDate: 0,
|
||||
oldEst: 0,
|
||||
mostRecent: 0,
|
||||
years: {},
|
||||
weeks: {},
|
||||
months: {},
|
||||
days: {},
|
||||
},
|
||||
logbook: {},
|
||||
properties: {},
|
||||
header: '',
|
||||
level: '',
|
||||
corpus: '',
|
||||
state: '',
|
||||
tags: [],
|
||||
children: [], // TODO list children tasks with a reference to the parent when level is superior to previous task
|
||||
tagsInherited: [], // TODO inherit tags
|
||||
dates: {
|
||||
havingDate: 0,
|
||||
havingNoDate: 0,
|
||||
oldEst: 0,
|
||||
mostRecent: 0,
|
||||
years: {},
|
||||
weeks: {},
|
||||
months: {},
|
||||
days: {},
|
||||
},
|
||||
logbook: {},
|
||||
properties: {},
|
||||
}
|
||||
|
||||
let isHeader = false;
|
||||
let isProperty = false;
|
||||
let isLogbook = false;
|
||||
let isFirst = true;
|
||||
let isHeader = false
|
||||
let isProperty = false
|
||||
let isLogbook = false
|
||||
let isFirst = true
|
||||
|
||||
// init first task object as empty clone
|
||||
let currentTask = Object.create(task);
|
||||
let currentTask = Object.create(task)
|
||||
|
||||
/**
|
||||
* add to tasks to export and refresh current task
|
||||
*/
|
||||
function addAndRefreshCurrentTask() {
|
||||
function addAndRefreshCurrentTask () {
|
||||
|
||||
makeWordsStatistics(currentTask.header.trim())
|
||||
makeWordsStatistics(currentTask.corpus.trim())
|
||||
makeWordsStatistics(currentTask.header.trim())
|
||||
makeWordsStatistics(currentTask.corpus.trim())
|
||||
|
||||
tasksObjectsForJsonExport.push(currentTask)
|
||||
// réinitialisation de tâche pour remplir de nouveau
|
||||
currentTask = Object.create(task);
|
||||
currentTask.dates = {};
|
||||
currentTask.tags = [];
|
||||
tasksObjectsForJsonExport.push(currentTask)
|
||||
// réinitialisation de tâche pour remplir de nouveau
|
||||
currentTask = Object.create(task)
|
||||
currentTask.dates = {}
|
||||
currentTask.tags = []
|
||||
}
|
||||
|
||||
function makeWordsStatistics(sentence) {
|
||||
let split = sentence.split(' ');
|
||||
if (split && split.length) {
|
||||
function makeWordsStatistics (sentence) {
|
||||
let split = sentence.split(' ')
|
||||
if (split && split.length) {
|
||||
|
||||
split.forEach(word => {
|
||||
if (!statistics.words[word]) {
|
||||
statistics.words[word] = 0
|
||||
}
|
||||
statistics.words[word]++
|
||||
})
|
||||
}
|
||||
split.forEach(word => {
|
||||
if (!statistics.words[word]) {
|
||||
statistics.words[word] = 0
|
||||
}
|
||||
statistics.words[word]++
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const dateStats = {
|
||||
created: 0,
|
||||
refiled: 0,
|
||||
closed: 0,
|
||||
cancelled: 0,
|
||||
scheduled: 0,
|
||||
deadline: 0,
|
||||
tasks_done: []
|
||||
created: 0,
|
||||
refiled: 0,
|
||||
closed: 0,
|
||||
cancelled: 0,
|
||||
scheduled: 0,
|
||||
deadline: 0,
|
||||
tasks_done: []
|
||||
}
|
||||
|
||||
function fillPeriodTime(periodStat, keyword){
|
||||
function fillPeriodTime (periodStat, keyword) {
|
||||
|
||||
if (keyword === 'CLOSED') {
|
||||
periodStat.closed++;
|
||||
}
|
||||
if (keyword === 'CREATED') {
|
||||
periodStat.created++;
|
||||
}
|
||||
if (keyword === 'Refiled') {
|
||||
periodStat.refiled++;
|
||||
}
|
||||
if (keyword === 'CANCELLED') {
|
||||
periodStat.cancelled++;
|
||||
}
|
||||
if (keyword === 'SCHEDULED') {
|
||||
periodStat.scheduled++;
|
||||
}
|
||||
if (keyword === 'DEADLINE') {
|
||||
periodStat.deadline++;
|
||||
}
|
||||
if (keyword === 'CLOSED') {
|
||||
periodStat.closed++
|
||||
}
|
||||
if (keyword === 'CREATED') {
|
||||
periodStat.created++
|
||||
}
|
||||
if (keyword === 'Refiled') {
|
||||
periodStat.refiled++
|
||||
}
|
||||
if (keyword === 'CANCELLED') {
|
||||
periodStat.cancelled++
|
||||
}
|
||||
if (keyword === 'SCHEDULED') {
|
||||
periodStat.scheduled++
|
||||
}
|
||||
if (keyword === 'DEADLINE') {
|
||||
periodStat.deadline++
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -171,60 +170,59 @@ function fillPeriodTime(periodStat, keyword){
|
|||
* @param keyword
|
||||
* @param dateFoundElement
|
||||
*/
|
||||
function statisticDateFill(keyword, dateFoundElement, header) {
|
||||
function statisticDateFill (keyword, dateFoundElement, header) {
|
||||
|
||||
// décompte par années
|
||||
// décompte par années
|
||||
|
||||
let convertedDate = new Date(dateFoundElement)
|
||||
let yearOfDate = convertedDate.getFullYear()
|
||||
let monthOfDate = yearOfDate + '-' + convertedDate.getMonth()
|
||||
// add zeros
|
||||
let convertedWeek = moment(convertedDate).week() < 10 ? '0' + moment(convertedDate).week() : moment(convertedDate).week()
|
||||
let weekOfDate = yearOfDate + '-' + convertedWeek
|
||||
let convertedDate = new Date(dateFoundElement)
|
||||
let yearOfDate = convertedDate.getFullYear()
|
||||
let monthOfDate = yearOfDate + '-' + convertedDate.getMonth()
|
||||
// add zeros
|
||||
let convertedWeek = moment(convertedDate).week() < 10 ? '0' + moment(convertedDate).week() : moment(convertedDate).week()
|
||||
let weekOfDate = yearOfDate + '-' + convertedWeek
|
||||
|
||||
let convertedMonth = convertedDate.getMonth() < 10 ? '0' + convertedDate.getMonth() : convertedDate.getMonth()
|
||||
let convertedDay = convertedDate.getDay() < 10 ? '0' + convertedDate.getDay() : convertedDate.getDay()
|
||||
let dayOfDate = convertedDate.getFullYear() + '-' + convertedMonth + '-' + convertedDay
|
||||
let convertedMonth = convertedDate.getMonth() < 10 ? '0' + convertedDate.getMonth() : convertedDate.getMonth()
|
||||
let convertedDay = convertedDate.getDay() < 10 ? '0' + convertedDate.getDay() : convertedDate.getDay()
|
||||
let dayOfDate = convertedDate.getFullYear() + '-' + convertedMonth + '-' + convertedDay
|
||||
|
||||
if (!statistics.dates.years[yearOfDate]) {
|
||||
statistics.dates.years[yearOfDate] = Object.create(dateStats)
|
||||
}
|
||||
fillPeriodTime(statistics.dates.years[yearOfDate], keyword)
|
||||
if (!statistics.dates.years[yearOfDate]) {
|
||||
statistics.dates.years[yearOfDate] = Object.create(dateStats)
|
||||
}
|
||||
fillPeriodTime(statistics.dates.years[yearOfDate], keyword)
|
||||
|
||||
// par année-semaine
|
||||
if (!statistics.dates.weeks[weekOfDate]) {
|
||||
statistics.dates.weeks[weekOfDate] = Object.create(dateStats)
|
||||
}
|
||||
fillPeriodTime(statistics.dates.weeks[weekOfDate], keyword)
|
||||
statistics.dates.weeks[weekOfDate].tasks_done.push(header)
|
||||
// décompte par mois
|
||||
if (!statistics.dates.months[monthOfDate]) {
|
||||
statistics.dates.months[monthOfDate] = Object.create(dateStats)
|
||||
}
|
||||
fillPeriodTime(statistics.dates.months[monthOfDate], keyword)
|
||||
// par année-semaine
|
||||
if (!statistics.dates.weeks[weekOfDate]) {
|
||||
statistics.dates.weeks[weekOfDate] = Object.create(dateStats)
|
||||
}
|
||||
fillPeriodTime(statistics.dates.weeks[weekOfDate], keyword)
|
||||
statistics.dates.weeks[weekOfDate].tasks_done.push(header)
|
||||
// décompte par mois
|
||||
if (!statistics.dates.months[monthOfDate]) {
|
||||
statistics.dates.months[monthOfDate] = Object.create(dateStats)
|
||||
}
|
||||
fillPeriodTime(statistics.dates.months[monthOfDate], keyword)
|
||||
|
||||
// décompte par jours
|
||||
// décompte par jours
|
||||
|
||||
if (!statistics.dates.days[dayOfDate]) {
|
||||
statistics.dates.days[dayOfDate] = Object.create(dateStats)
|
||||
}
|
||||
fillPeriodTime(statistics.dates.days[dayOfDate], keyword)
|
||||
statistics.dates.days[dayOfDate].tasks_done.push(header)
|
||||
if (!statistics.dates.days[dayOfDate]) {
|
||||
statistics.dates.days[dayOfDate] = Object.create(dateStats)
|
||||
}
|
||||
fillPeriodTime(statistics.dates.days[dayOfDate], keyword)
|
||||
statistics.dates.days[dayOfDate].tasks_done.push(header)
|
||||
}
|
||||
|
||||
function findOldestDate(currentDate) {
|
||||
// trouver la plus ancienne date
|
||||
function findOldestDate (currentDate) {
|
||||
// trouver la plus ancienne date
|
||||
|
||||
|
||||
if (!statistics.dates.oldEst) {
|
||||
statistics.dates.oldEst = currentDate;
|
||||
} else {
|
||||
var beginningTime = moment(statistics.dates.oldEst);
|
||||
var endTime = moment(currentDate);
|
||||
if (!beginningTime.isBefore(endTime)) {
|
||||
statistics.dates.oldEst = currentDate;
|
||||
}
|
||||
}
|
||||
if (!statistics.dates.oldEst) {
|
||||
statistics.dates.oldEst = currentDate
|
||||
} else {
|
||||
var beginningTime = moment(statistics.dates.oldEst)
|
||||
var endTime = moment(currentDate)
|
||||
if (!beginningTime.isBefore(endTime)) {
|
||||
statistics.dates.oldEst = currentDate
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**********************
|
||||
|
|
@ -232,208 +230,199 @@ function findOldestDate(currentDate) {
|
|||
*********************/
|
||||
fs.readFile(sourceFilePath, 'utf8', function (err, data) {
|
||||
|
||||
if (err) {
|
||||
return console.log(err)
|
||||
}
|
||||
console.log(' parsing...')
|
||||
// parcourir chaque ligne du fichier org
|
||||
let everyline = data.split('\n')
|
||||
|
||||
if (err) {
|
||||
return console.log(err);
|
||||
}
|
||||
console.log(" parsing...")
|
||||
// parcourir chaque ligne du fichier org
|
||||
let everyline = data.split('\n');
|
||||
// trouver les entêtes toutes les lignes qui commencent par * et espace.
|
||||
|
||||
// trouver les entêtes toutes les lignes qui commencent par * et espace.
|
||||
everyline.forEach((line) => {
|
||||
|
||||
everyline.forEach((line) => {
|
||||
// gérer la création d'objets définissant les tâches et leurs propriétés
|
||||
if (line.match(/^\*+? /)) {
|
||||
// add last task to export list
|
||||
if (!isFirst) {
|
||||
|
||||
addAndRefreshCurrentTask()
|
||||
} else {
|
||||
isFirst = false
|
||||
}
|
||||
|
||||
// gérer la création d'objets définissant les tâches et leurs propriétés
|
||||
if (line.match(/^\*+? /)) {
|
||||
// add last task to export list
|
||||
if (!isFirst) {
|
||||
isHeader = true
|
||||
// compter les étoiles pour trouver le niveau du header
|
||||
if (line.match(/\*/g)) {
|
||||
|
||||
addAndRefreshCurrentTask();
|
||||
} else {
|
||||
isFirst = false;
|
||||
}
|
||||
let match = line.match(/\*/g)
|
||||
currentTask.level = match.length
|
||||
}
|
||||
|
||||
isHeader = true;
|
||||
// compter les étoiles pour trouver le niveau du header
|
||||
if (line.match(/\*/g)) {
|
||||
// create a new task
|
||||
|
||||
let match = line.match(/\*/g);
|
||||
currentTask.level = match.length
|
||||
}
|
||||
headers.push(cleanHeader(line))
|
||||
currentTask.header = cleanHeader(line)
|
||||
stateKeywordList.forEach(keyword => {
|
||||
let keywordIsFound = lineHasKeyword(line, keyword)
|
||||
|
||||
// create a new task
|
||||
if (keywordIsFound) {
|
||||
currentTask.state = keyword
|
||||
}
|
||||
})
|
||||
|
||||
// trouver les tags
|
||||
let tagsList = line.match(/\:([\w\_]*)\:/g)
|
||||
if (tagsList) {
|
||||
tagsList = tagsList[0]
|
||||
let tagList = tagsList.split(':')
|
||||
if (tagList.length) {
|
||||
|
||||
headers.push(cleanHeader(line))
|
||||
currentTask.header = cleanHeader(line);
|
||||
stateKeywordList.forEach(keyword => {
|
||||
let keywordIsFound = lineHasKeyword(line, keyword)
|
||||
tagList.forEach(tag => {
|
||||
if (tag.length > 1) {
|
||||
|
||||
if (keywordIsFound) {
|
||||
currentTask.state = keyword
|
||||
}
|
||||
})
|
||||
if (!statistics.tags[tag]) {
|
||||
statistics.tags[tag] = 0
|
||||
}
|
||||
statistics.tags[tag]++
|
||||
|
||||
currentTask.tags.push(tag)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// trouver les tags
|
||||
let tagsList = line.match(/\:([\w\_]*)\:/g)
|
||||
if (tagsList) {
|
||||
tagsList = tagsList[0];
|
||||
let tagList = tagsList.split(':');
|
||||
if (tagList.length) {
|
||||
// ------------- fin des recherches dans la ligne de Header -------------
|
||||
} else {
|
||||
isHeader = false
|
||||
}
|
||||
// examen des lignes de corps de tâche, ou de corps de section suite au header.
|
||||
// classer les dates de création, cloture, et de logbook
|
||||
let dateFound = searchDate(line)
|
||||
if (dateFound) {
|
||||
/**
|
||||
* we have found a date in the current line
|
||||
*/
|
||||
|
||||
tagList.forEach(tag => {
|
||||
if (tag.length > 1) {
|
||||
statistics.dates.havingDate += 1
|
||||
|
||||
if (!statistics.tags[tag]) {
|
||||
statistics.tags[tag] = 0
|
||||
}
|
||||
statistics.tags[tag]++
|
||||
dateKeywordList.forEach(keyword => {
|
||||
if (lineHasSubstring(line, keyword)) {
|
||||
if (!currentTask.dates[keyword]) {
|
||||
currentTask.dates[keyword] = ''
|
||||
}
|
||||
|
||||
currentTask.tags.push(tag)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
let convertedDate = dateFound[0].substring(0, 10)
|
||||
|
||||
if (dateFound[0].length === 15) {
|
||||
// sans heure: "2022-12-21 mer."
|
||||
convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd')
|
||||
|
||||
// ------------- fin des recherches dans la ligne de Header -------------
|
||||
} else {
|
||||
isHeader = false;
|
||||
}
|
||||
// examen des lignes de corps de tâche, ou de corps de section suite au header.
|
||||
// classer les dates de création, cloture, et de logbook
|
||||
let dateFound = searchDate(line)
|
||||
if (dateFound) {
|
||||
/**
|
||||
* we have found a date in the current line
|
||||
*/
|
||||
} else if (dateFound[0].length === 21) {
|
||||
// avec heure: "2022-11-01 mar. 00:44"
|
||||
convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd HH:mm')
|
||||
}
|
||||
let formattedDate = moment(convertedDate).format()
|
||||
// console.log('currentTask.header', currentTask.header)
|
||||
statisticDateFill(keyword, convertedDate, currentTask.header)
|
||||
findOldestDate(convertedDate)
|
||||
currentTask.dates[keyword] = formattedDate
|
||||
|
||||
statistics.dates.havingDate += 1;
|
||||
} else {
|
||||
// console.log('keyword', keyword)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
statistics.dates.havingNoDate += 1
|
||||
if (
|
||||
line.indexOf(dateKeywordList) !== -1 &&
|
||||
line.indexOf(stateKeywordList) !== -1 &&
|
||||
line.indexOf(sectionKeywordList) !== -1
|
||||
) {
|
||||
|
||||
dateKeywordList.forEach(keyword => {
|
||||
if (lineHasSubstring(line, keyword)) {
|
||||
if (!currentTask.dates[keyword]) {
|
||||
currentTask.dates[keyword] = '';
|
||||
}
|
||||
// ajouter le corps complet de la section après le header
|
||||
if (line.length && !isHeader) {
|
||||
|
||||
let convertedDate = dateFound[0].substring(0, 10);
|
||||
let cleanedLine = line.replace(/\s\s/g, ' ')
|
||||
cleanedLine = line.replace(/ {2,}/g, ' ')
|
||||
|
||||
if (dateFound[0].length == 15) {
|
||||
// sans heure: "2022-12-21 mer."
|
||||
convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd')
|
||||
cleanedLine = cleanedLine.trim()
|
||||
|
||||
} else if (dateFound[0].length == 21) {
|
||||
// avec heure: "2022-11-01 mar. 00:44"
|
||||
convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd HH:mm')
|
||||
}
|
||||
let formattedDate = moment(convertedDate).format()
|
||||
console.log('currentTask.header', currentTask.header)
|
||||
statisticDateFill(keyword, convertedDate, currentTask.header)
|
||||
findOldestDate(convertedDate)
|
||||
currentTask.dates[keyword] = formattedDate;
|
||||
|
||||
|
||||
} else {
|
||||
// console.log('keyword', keyword)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
statistics.dates.havingNoDate += 1;
|
||||
if (
|
||||
line.indexOf(dateKeywordList) !== -1 &&
|
||||
line.indexOf(stateKeywordList) !== -1 &&
|
||||
line.indexOf(sectionKeywordList) !== -1
|
||||
) {
|
||||
|
||||
// ajouter le corps complet de la section après le header
|
||||
if (line.length && !isHeader) {
|
||||
|
||||
let cleanedLine = line.replace(/\s\s/g, ' ');
|
||||
cleanedLine = line.replace(/ {2,}/g, ' ')
|
||||
|
||||
cleanedLine = cleanedLine.trim()
|
||||
|
||||
currentTask.corpus += `${cleanedLine}
|
||||
currentTask.corpus += `${cleanedLine}
|
||||
`
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// ajouter la dernière tâche parsée
|
||||
addAndRefreshCurrentTask()
|
||||
|
||||
// ajouter la dernière tâche parsée
|
||||
addAndRefreshCurrentTask();
|
||||
console.log('tasks : ', tasksObjectsForJsonExport.length)
|
||||
console.log(tada + ' parsing fini ' + tada)
|
||||
|
||||
console.log(tada+ " parsing fini "+ tada)
|
||||
// ranger par valeur décroissante les tags
|
||||
|
||||
// ranger par valeur décroissante les tags
|
||||
let sorted_stats = []
|
||||
// rangement par valeur et par date
|
||||
console.log('write file ', outputAbsolutePath, outputFileNameJson)
|
||||
statistics.dates.years = sortByKey(statistics.dates.years)
|
||||
statistics.dates.weeks = sortByKey(statistics.dates.weeks)
|
||||
statistics.dates.months = sortByKey(statistics.dates.months)
|
||||
statistics.dates.days = sortByKey(statistics.dates.days)
|
||||
|
||||
let sorted_stats = [];
|
||||
// rangement par valeur et par date
|
||||
console.log('write file ', outputAbsolutePath, outputFileNameJson);
|
||||
statistics.dates.years = sortByKey(statistics.dates.years)
|
||||
statistics.dates.weeks = sortByKey(statistics.dates.weeks)
|
||||
statistics.dates.months = sortByKey(statistics.dates.months)
|
||||
statistics.dates.days = sortByKey(statistics.dates.days)
|
||||
statistics = sortByKey(statistics)
|
||||
|
||||
statistics = sortByKey(statistics)
|
||||
const jsonContent = {
|
||||
statistics: {
|
||||
lines_count: everyline.length,
|
||||
headers_count: headers.length,
|
||||
statistics
|
||||
},
|
||||
meta_data: {
|
||||
author: '@tykayn@mastodon.Cipherbliss.com',
|
||||
generated_at: new Date(),
|
||||
generated_from_file: sourceFilePath + sourceFileName,
|
||||
sources: 'https://forge.chapril.org/tykayn/org-report-stats.git'
|
||||
},
|
||||
tasks_list: tasksObjectsForJsonExport
|
||||
}
|
||||
|
||||
if (writeJsonAfterParse) {
|
||||
|
||||
const jsonContent = {
|
||||
statistics: {
|
||||
lines_count: everyline.length,
|
||||
headers_count: headers.length,
|
||||
statistics
|
||||
},
|
||||
meta_data: {
|
||||
author: '@tykayn@mastodon.Cipherbliss.com',
|
||||
generated_at: new Date(),
|
||||
generated_from_file: sourceFilePath + sourceFileName,
|
||||
sources: 'https://forge.chapril.org/tykayn/org-report-stats.git'
|
||||
},
|
||||
tasks_list: tasksObjectsForJsonExport
|
||||
}
|
||||
|
||||
|
||||
if (writeJsonAfterParse) {
|
||||
|
||||
writeFileInOuputFolderFromJsonObject(outputFileNameJson, jsonContent);
|
||||
}
|
||||
writeFileInOuputFolderFromJsonObject(outputFileNameJson, jsonContent)
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
function lineHasKeyword (line, keyword = 'TODO') {
|
||||
|
||||
function lineHasKeyword(line, keyword = 'TODO') {
|
||||
|
||||
let isFound = (line.indexOf('* ' + keyword) !== -1)
|
||||
if (isFound) {
|
||||
createNewHeaderKind(keyword)
|
||||
headersByKind[keyword].push(line);
|
||||
if (!statistics[keyword]) {
|
||||
statistics[keyword] = 0
|
||||
}
|
||||
statistics[keyword]++
|
||||
}
|
||||
return isFound;
|
||||
let isFound = (line.indexOf('* ' + keyword) !== -1)
|
||||
if (isFound) {
|
||||
createNewHeaderKind(keyword)
|
||||
headersByKind[keyword].push(line)
|
||||
if (!statistics[keyword]) {
|
||||
statistics[keyword] = 0
|
||||
}
|
||||
statistics[keyword]++
|
||||
}
|
||||
return isFound
|
||||
}
|
||||
|
||||
function lineHasSubstring(line, keyword) {
|
||||
let isFound = (line.indexOf(keyword) !== -1)
|
||||
if (!statistics[keyword]) {
|
||||
statistics[keyword] = 0
|
||||
}
|
||||
statistics[keyword]++
|
||||
function lineHasSubstring (line, keyword) {
|
||||
let isFound = (line.indexOf(keyword) !== -1)
|
||||
if (!statistics[keyword]) {
|
||||
statistics[keyword] = 0
|
||||
}
|
||||
statistics[keyword]++
|
||||
|
||||
return isFound
|
||||
return isFound
|
||||
}
|
||||
|
||||
function createNewHeaderKind(keyword) {
|
||||
if (!headersByKind[keyword]) {
|
||||
headersByKind[keyword] = [];
|
||||
}
|
||||
function createNewHeaderKind (keyword) {
|
||||
if (!headersByKind[keyword]) {
|
||||
headersByKind[keyword] = []
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -443,22 +432,22 @@ function createNewHeaderKind(keyword) {
|
|||
* @param line
|
||||
* @returns {*}
|
||||
*/
|
||||
function searchDate(line) {
|
||||
// return line.match(/[(\d{4}\-\d{2}\-\d{2} ?\d{2}?\:?\d{2}?\:?\d{2}?)(\d{4}\-\d{2}\-\d{2})]/)
|
||||
let simpleDay = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.?/)
|
||||
let simpleDayHour = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.? \d{2}\:\d{2}/)
|
||||
let simpleDayHourSec = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.? \d{2}\:\d{2}\:\d{2}/)
|
||||
function searchDate (line) {
|
||||
// return line.match(/[(\d{4}\-\d{2}\-\d{2} ?\d{2}?\:?\d{2}?\:?\d{2}?)(\d{4}\-\d{2}\-\d{2})]/)
|
||||
let simpleDay = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.?/)
|
||||
let simpleDayHour = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.? \d{2}\:\d{2}/)
|
||||
let simpleDayHourSec = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.? \d{2}\:\d{2}\:\d{2}/)
|
||||
|
||||
if (simpleDayHourSec) {
|
||||
return simpleDayHourSec;
|
||||
}
|
||||
if (simpleDayHourSec) {
|
||||
return simpleDayHourSec
|
||||
}
|
||||
|
||||
if (simpleDayHour) {
|
||||
return simpleDayHour;
|
||||
}
|
||||
if (simpleDay) {
|
||||
return simpleDay;
|
||||
}
|
||||
if (simpleDayHour) {
|
||||
return simpleDayHour
|
||||
}
|
||||
if (simpleDay) {
|
||||
return simpleDay
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -466,17 +455,17 @@ function searchDate(line) {
|
|||
* get the cleaned content of the header
|
||||
* @param line
|
||||
*/
|
||||
function cleanHeader(line) {
|
||||
function cleanHeader (line) {
|
||||
|
||||
line = '' + line;
|
||||
stateKeywordList.forEach(keyword => {
|
||||
line = line.replace(keyword, '')
|
||||
})
|
||||
line = line.replace(/\** /, '');
|
||||
line = line.replace(/\[.*\]/g, '');
|
||||
line = line.replace(/\:.*\:/g, '');
|
||||
line = line.replace(' ', '');
|
||||
return line.trim();
|
||||
line = '' + line
|
||||
stateKeywordList.forEach(keyword => {
|
||||
line = line.replace(keyword, '')
|
||||
})
|
||||
line = line.replace(/\** /, '')
|
||||
line = line.replace(/\[.*\]/g, '')
|
||||
line = line.replace(/\:.*\:/g, '')
|
||||
line = line.replace(' ', '')
|
||||
return line.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -484,14 +473,14 @@ function cleanHeader(line) {
|
|||
* @param objectStuff
|
||||
* @returns {{}}
|
||||
*/
|
||||
function sortByKey(objectStuff) {
|
||||
return Object.keys(objectStuff).sort().reduce(
|
||||
(obj, key) => {
|
||||
obj[key] = objectStuff[key];
|
||||
return obj;
|
||||
},
|
||||
{}
|
||||
);
|
||||
function sortByKey (objectStuff) {
|
||||
return Object.keys(objectStuff).sort().reduce(
|
||||
(obj, key) => {
|
||||
obj[key] = objectStuff[key]
|
||||
return obj
|
||||
},
|
||||
{}
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -499,32 +488,32 @@ function sortByKey(objectStuff) {
|
|||
* @param literalobject
|
||||
* @returns {any}
|
||||
*/
|
||||
function sortByValue(literalobject) {
|
||||
function sortByValue (literalobject) {
|
||||
|
||||
let sortable = [];
|
||||
for (var keyName in literalobject) {
|
||||
sortable[keyName] = literalobject[keyName];
|
||||
}
|
||||
// return literalobject
|
||||
return sortable.sort(function (a, b) {
|
||||
return b[1] - a[1];
|
||||
});
|
||||
let sortable = []
|
||||
for (var keyName in literalobject) {
|
||||
sortable[keyName] = literalobject[keyName]
|
||||
}
|
||||
// return literalobject
|
||||
return sortable.sort(function (a, b) {
|
||||
return b[1] - a[1]
|
||||
})
|
||||
}
|
||||
|
||||
export async function writeFileInOuputFolderFromJsonObject(fileName, jsonObjectThing) {
|
||||
export async function writeFileInOuputFolderFromJsonObject (fileName, jsonObjectThing) {
|
||||
|
||||
// console.log('statistics.dates', statistics.dates)
|
||||
// console.log('statistics.dates', statistics.dates)
|
||||
|
||||
return await fs.writeFile(
|
||||
`${outputAbsolutePath}${fileName}`,
|
||||
JSON.stringify(jsonObjectThing, null,2),
|
||||
"utf8",
|
||||
(err) => {
|
||||
if (err) {
|
||||
console.log(`Error writing file: ${err}`);
|
||||
} else {
|
||||
console.log(`\n ${gift} File ${fileName} is written successfully!`);
|
||||
}
|
||||
}
|
||||
);
|
||||
return await fs.writeFile(
|
||||
`${outputAbsolutePath}${fileName}`,
|
||||
JSON.stringify(jsonObjectThing, null, 2),
|
||||
'utf8',
|
||||
(err) => {
|
||||
if (err) {
|
||||
console.log(`Error writing file: ${err}`)
|
||||
} else {
|
||||
console.log(`\n ${gift} File ${fileName} is written successfully!`)
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
|
|
@ -12,11 +12,12 @@ import { writeFileInOuputFolderFromJsonObject } from './utils.mjs'
|
|||
**********************/
|
||||
|
||||
// const sourceFileBaseName = 'sms-20180423162531'
|
||||
const sourceFileBaseName = 'sms'
|
||||
const sourceFileBaseName = 'sms-20230921224605'
|
||||
const sourceFileName = `${sourceFileBaseName}.xml`
|
||||
const outputFileJson = `${sourceFileBaseName}.json`
|
||||
const outputFileCsv = `${sourceFileBaseName}.tsv`
|
||||
const outputFileCsv = `${sourceFileBaseName}.csv`
|
||||
const folder_base = '/home/tykayn/Nextcloud/ressources/social sorting/'
|
||||
const csv_separator = ';'
|
||||
|
||||
const outputFileJsonPathFull = `${folder_base}output/${outputFileJson}`
|
||||
const sourceFileJson = `${folder_base}output/${outputFileJson}`
|
||||
|
|
@ -54,46 +55,49 @@ function convertJsonToCsvAndPersistTSVFile (jsonFilePath, outputFileTSVName) {
|
|||
|
||||
console.log('convert', folder_base+'output/' + jsonFilePath)
|
||||
fs.readFile(folder_base+'output/' + jsonFilePath, 'utf8', function (err, data) {
|
||||
|
||||
data = JSON.parse(data)
|
||||
console.log('data', data.length)
|
||||
if (data) {
|
||||
|
||||
const events = [
|
||||
|
||||
'amount\t' +
|
||||
'content\t' +
|
||||
'description\t' +
|
||||
'destination\t' +
|
||||
'end\t' +
|
||||
'kind of activity\t' +
|
||||
'person\t' +
|
||||
'place\t' +
|
||||
'source\t' +
|
||||
'start\t' +
|
||||
'unique id\t' +
|
||||
'url\t'
|
||||
'amount' + csv_separator + ''+
|
||||
'content' + csv_separator +''+
|
||||
'description' + csv_separator +
|
||||
'destination' + csv_separator +
|
||||
'end' + csv_separator +
|
||||
'kind of activity' + csv_separator +
|
||||
'person' + csv_separator +
|
||||
'place' + csv_separator +
|
||||
'source' + csv_separator +
|
||||
'start' + csv_separator +
|
||||
'unique id' + csv_separator +
|
||||
'url' + csv_separator
|
||||
]
|
||||
data = JSON.parse(data)
|
||||
// console.log('data.length', data.length)
|
||||
console.log('data', Object.keys(data))
|
||||
let smslist = data['elements'][1]['elements']
|
||||
|
||||
// console.log('data', data)
|
||||
// console.log('data other ', data)
|
||||
console.log('data.length', data['elements'].length)
|
||||
console.log('data name', data['elements'][2]['elements'].length)
|
||||
let smslist = data['elements'][2]['elements']
|
||||
// let smslist = data['smses']
|
||||
if (smslist) {
|
||||
smslist.forEach(item => {
|
||||
|
||||
// convert all fields to common event description
|
||||
events.push(
|
||||
'\t' +
|
||||
item.attributes.body.replace('\n', ' ') + '\t' +
|
||||
'sms ' + item.attributes.address + ' le ' + item.attributes.readable_date + '\t' +
|
||||
item.attributes.address + '\t' +
|
||||
'' + '\t' +
|
||||
'' + '\t' +
|
||||
item.attributes.contact_name + '\t' +
|
||||
'' + '\t' +
|
||||
'' + '\t' +
|
||||
'' + '\t' +
|
||||
'' + '\t' +
|
||||
'' + '\t'
|
||||
''+csv_separator +
|
||||
item.attributes.body + csv_separator +
|
||||
'sms ' + item.attributes.address + ' le ' + item.attributes.readable_date + csv_separator +
|
||||
item.attributes.address + csv_separator +
|
||||
'' + csv_separator +
|
||||
'' + csv_separator +
|
||||
item.attributes.contact_name + csv_separator +
|
||||
'' + csv_separator +
|
||||
'' + csv_separator +
|
||||
'' + csv_separator +
|
||||
'' + csv_separator +
|
||||
'' + csv_separator
|
||||
)
|
||||
})
|
||||
// // console.log('events', events)
|
||||
|
|
@ -122,10 +126,11 @@ function openSourceFile () {
|
|||
fs.readFile(sourceFilePath, 'utf8', function (err, data) {
|
||||
// // console.log('data', data)
|
||||
var jsonConversion = convert.xml2json(data)
|
||||
console.log('jsonConversion keys', Object.keys(jsonConversion))
|
||||
// console.log('jsonConversion 0', Object.keys(jsonConversion['elements']))
|
||||
// console.log('jsonConversion.length', parseFloat(jsonConversion.length / 1024 / 1024, 2), 'Mo')
|
||||
|
||||
writeFileInOuputFolderFromJsonObject(outputFileJson, JSON.parse( jsonConversion))
|
||||
// writeFileInOuputFolderFromJsonObject(outputFileJson, JSON.parse( jsonConversion))
|
||||
writeFileInOuputFolderFromJsonObject(outputFileJson, JSON.parse(jsonConversion))
|
||||
convertJsonToCsvAndPersistTSVFile(outputFileJson, outputFileCsv)
|
||||
|
||||
})
|
||||
|
|
|
|||
|
|
@ -20,11 +20,11 @@ export async function writeFileInOuputFolderFromJsonObject (fileName, jsonObject
|
|||
// console.log('statistics.dates', statistics.dates)
|
||||
let fullpath = `${outputAbsolutePath}${fileName}`
|
||||
|
||||
return await fs.writeFile(
|
||||
return await fs.writeFileSync(
|
||||
fullpath,
|
||||
JSON.stringify(jsonObjectThing, null, 2),
|
||||
// JSON.stringify(jsonObjectThing).replace('\n',''),
|
||||
'utf8',
|
||||
// JSON.stringify(jsonObjectThing, null, 2),
|
||||
JSON.stringify(jsonObjectThing).replace('\n',''),
|
||||
'UTF8',
|
||||
(err) => {
|
||||
if (err) {
|
||||
console.log(`Error writing file: ${err}`)
|
||||
|
|
|
|||
Loading…
Reference in New Issue