tykayn
/
org-report-stats
mirror of https://forge.chapril.org/tykayn/org-report-stats.git
1
0
Fork 0
Code Issues Releases Activity

hop

This commit is contained in:
Tykayn 2023-09-22 10:16:46 +02:00 committed by tykayn
parent 37fc569fa3
commit 8c603e291e
5 changed files with 402 additions and 411 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
README.md
View File

@ -16,10 +16,10 @@ pour le lancer: npm start.
* avoir Nodejs en version stable et installer les modules node.
# Utilisation
## Configuration
## Configuration
Vérifiez les variables de chemin dans **parse_orgmode_to_json.mjs**, surtout outputAbsolutePath pour avoir un lieu de génération de fichier.
## compiler les fichiers org et les convertir en json
## compiler les fichiers org et les convertir en json
Il suffit de lancer cette commande du Makefile
```bash
make convert
@ -28,8 +28,8 @@ Elle réalise la concaténation des fichiers tasks.org et tasks.org_archive fait
la sortie est générée dans "outputAbsolutePath" définie dans "parse_orgmode_to_json.mjs"
## Données générées
### Des statistiques
## Données générées
### Des statistiques
* Les nombres et le type de tâches agrégées par semaine, mois et années.
* le nombre de tâches avec une date.
@ -41,20 +41,17 @@ la sortie est générée dans "outputAbsolutePath" définie dans "parse_orgmode_
* Les nombres de fois que des mots sont utilisés dans les tâches.
* des données sur l'auteur, la date de génération du json, le fichier source .org ayant servi à la conversion.
###
## voir le rendu
## voir le rendu
Lancer l'exécution du fichier app avec node, et consulter l'output html.
```bash
npm start
```
## Memacs
## Memacs
Les scripts inspirés de Memacs (le package python-pip) sont dans le dossier **converters**, ils permettent de convertir des fichiers exportés de divers outils numériques en feuilles de calcul pour avoir un historique général d'activités simple à lire pour les humains, et facile à analyser pour d'autres scripts.
# feuille de route TODO
Proposer une sortie json qui soit du même format que l'export ox-json d'emacs.
- Proposer une sortie json qui soit du même format que l'export ox-json d'emacs.
- Inclure les sous tâches aux catégories
# DONE

191
parse_orgmode_to_json.mjs
View File

@ -2,29 +2,28 @@
* convertir un fichier .org vers des données structurées en json
* @type {*}
*/
import fs from 'node-fs';
import moment from 'moment';
import * as emoji from "node-emoji";
import fs from 'node-fs'
import moment from 'moment'
import * as emoji from 'node-emoji'
/**********************
* initialize configs
**********************/
const sourceFileName = 'all_tasks.org'
const sourceFilePath = './sources/' + sourceFileName;
const outputAbsolutePath = '~/Nextcloud/ressources/social sorting/output/';
const outputFileNameJson = 'export_' + sourceFileName + '_parsed.json';
const sourceFilePath = './sources/' + sourceFileName
const outputAbsolutePath = '~/Nextcloud/ressources/social sorting/output/'
const outputFileNameJson = 'export_' + sourceFileName + '_parsed.json'
let headers = []
let tasksObjectsForJsonExport = []
let headersByKind = {}
let writeJsonAfterParse = false;
writeJsonAfterParse = true;
let writeJsonAfterParse = false
writeJsonAfterParse = true
moment.locale('FR')
const tada = emoji.get("tada");
const gift = emoji.get("gift");
const tada = emoji.get('tada')
const gift = emoji.get('gift')
/**************************************************************
* fetch the source orgmode file to read its contents
@ -35,23 +34,23 @@ if (!sourceFilePath) {
console.error('pas de fichier à ouvrir')
}
fs.stat(sourceFilePath, function (err, stat) {
if (err == null) {
console.log(`File ${sourceFilePath} exists`);
if (err === null) {
console.log(`File ${sourceFilePath} exists`)
} else if (err.code === 'ENOENT') {
// file does not exist
console.error(`le fichier ${sourceFilePath} est introuvable. Impossible d en extraire des infos.`, err);
console.error(`le fichier ${sourceFilePath} est introuvable. Impossible d en extraire des infos.`, err)
} else {
console.log('Some other error: ', err.code);
console.log('Some other error: ', err.code)
}
});
})
/**********************
* search elements
*********************/
let stateKeywordList = ['SOMEDAY', 'NEXT', 'TODO', 'CANCELLED', 'DONE', 'WAITING'];
let dateKeywordList = ['CREATED', 'SCHEDULED', 'DEADLINE', 'CLOSED', 'Refiled'];
let sectionKeywordList = ['PROPERTIES', 'LOGBOOK', 'END'];
let stateKeywordList = ['SOMEDAY', 'NEXT', 'TODO', 'CANCELLED', 'DONE', 'WAITING']
let dateKeywordList = ['CREATED', 'SCHEDULED', 'DEADLINE', 'CLOSED', 'Refiled']
let sectionKeywordList = ['PROPERTIES', 'LOGBOOK', 'END']
let propertiesSection = {} // TODO properties listing
let logBookSection = {} // TODO logbook listing
@ -77,10 +76,10 @@ let headerKeywordSearch = '[' + stateKeywordList.join('|') + ']'
* @type {{level: string, header: string, dates: {CREATED: string, DONE: string, REFILED: string}, state: string, content: string, properties: {}, tags: [], tagsInherited: []}}
*/
let task = {
header: "",
level: "",
corpus: "",
state: "",
header: '',
level: '',
corpus: '',
state: '',
tags: [],
children: [], // TODO list children tasks with a reference to the parent when level is superior to previous task
tagsInherited: [], // TODO inherit tags
@ -98,13 +97,13 @@ let task = {
properties: {},
}
let isHeader = false;
let isProperty = false;
let isLogbook = false;
let isFirst = true;
let isHeader = false
let isProperty = false
let isLogbook = false
let isFirst = true
// init first task object as empty clone
let currentTask = Object.create(task);
let currentTask = Object.create(task)
/**
* add to tasks to export and refresh current task
@ -116,13 +115,13 @@ function addAndRefreshCurrentTask() {
tasksObjectsForJsonExport.push(currentTask)
// réinitialisation de tâche pour remplir de nouveau
currentTask = Object.create(task);
currentTask.dates = {};
currentTask.tags = [];
currentTask = Object.create(task)
currentTask.dates = {}
currentTask.tags = []
}
function makeWordsStatistics (sentence) {
let split = sentence.split(' ');
let split = sentence.split(' ')
if (split && split.length) {
split.forEach(word => {
@ -147,22 +146,22 @@ const dateStats = {
function fillPeriodTime (periodStat, keyword) {
if (keyword === 'CLOSED') {
periodStat.closed++;
periodStat.closed++
}
if (keyword === 'CREATED') {
periodStat.created++;
periodStat.created++
}
if (keyword === 'Refiled') {
periodStat.refiled++;
periodStat.refiled++
}
if (keyword === 'CANCELLED') {
periodStat.cancelled++;
periodStat.cancelled++
}
if (keyword === 'SCHEDULED') {
periodStat.scheduled++;
periodStat.scheduled++
}
if (keyword === 'DEADLINE') {
periodStat.deadline++;
periodStat.deadline++
}
}
@ -215,14 +214,13 @@ function statisticDateFill(keyword, dateFoundElement, header) {
function findOldestDate (currentDate) {
// trouver la plus ancienne date
if (!statistics.dates.oldEst) {
statistics.dates.oldEst = currentDate;
statistics.dates.oldEst = currentDate
} else {
var beginningTime = moment(statistics.dates.oldEst);
var endTime = moment(currentDate);
var beginningTime = moment(statistics.dates.oldEst)
var endTime = moment(currentDate)
if (!beginningTime.isBefore(endTime)) {
statistics.dates.oldEst = currentDate;
statistics.dates.oldEst = currentDate
}
}
}
@ -232,42 +230,39 @@ function findOldestDate(currentDate) {
*********************/
fs.readFile(sourceFilePath, 'utf8', function (err, data) {
if (err) {
return console.log(err);
return console.log(err)
}
console.log(" parsing...")
console.log(' parsing...')
// parcourir chaque ligne du fichier org
let everyline = data.split('\n');
let everyline = data.split('\n')
// trouver les entêtes toutes les lignes qui commencent par * et espace.
everyline.forEach((line) => {
// gérer la création d'objets définissant les tâches et leurs propriétés
if (line.match(/^\*+? /)) {
// add last task to export list
if (!isFirst) {
addAndRefreshCurrentTask();
addAndRefreshCurrentTask()
} else {
isFirst = false;
isFirst = false
}
isHeader = true;
isHeader = true
// compter les étoiles pour trouver le niveau du header
if (line.match(/\*/g)) {
let match = line.match(/\*/g);
let match = line.match(/\*/g)
currentTask.level = match.length
}
// create a new task
headers.push(cleanHeader(line))
currentTask.header = cleanHeader(line);
currentTask.header = cleanHeader(line)
stateKeywordList.forEach(keyword => {
let keywordIsFound = lineHasKeyword(line, keyword)
@ -276,12 +271,11 @@ fs.readFile(sourceFilePath, 'utf8', function (err, data) {
}
})
// trouver les tags
let tagsList = line.match(/\:([\w\_]*)\:/g)
if (tagsList) {
tagsList = tagsList[0];
let tagList = tagsList.split(':');
tagsList = tagsList[0]
let tagList = tagsList.split(':')
if (tagList.length) {
tagList.forEach(tag => {
@ -298,10 +292,9 @@ fs.readFile(sourceFilePath, 'utf8', function (err, data) {
}
}
// ------------- fin des recherches dans la ligne de Header -------------
} else {
isHeader = false;
isHeader = false
}
// examen des lignes de corps de tâche, ou de corps de section suite au header.
// classer les dates de création, cloture, et de logbook
@ -311,37 +304,36 @@ fs.readFile(sourceFilePath, 'utf8', function (err, data) {
* we have found a date in the current line
*/
statistics.dates.havingDate += 1;
statistics.dates.havingDate += 1
dateKeywordList.forEach(keyword => {
if (lineHasSubstring(line, keyword)) {
if (!currentTask.dates[keyword]) {
currentTask.dates[keyword] = '';
currentTask.dates[keyword] = ''
}
let convertedDate = dateFound[0].substring(0, 10);
let convertedDate = dateFound[0].substring(0, 10)
if (dateFound[0].length == 15) {
if (dateFound[0].length === 15) {
// sans heure: "2022-12-21 mer."
convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd')
} else if (dateFound[0].length == 21) {
} else if (dateFound[0].length === 21) {
// avec heure: "2022-11-01 mar. 00:44"
convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd HH:mm')
}
let formattedDate = moment(convertedDate).format()
console.log('currentTask.header', currentTask.header)
// console.log('currentTask.header', currentTask.header)
statisticDateFill(keyword, convertedDate, currentTask.header)
findOldestDate(convertedDate)
currentTask.dates[keyword] = formattedDate;
currentTask.dates[keyword] = formattedDate
} else {
// console.log('keyword', keyword)
}
})
} else {
statistics.dates.havingNoDate += 1;
statistics.dates.havingNoDate += 1
if (
line.indexOf(dateKeywordList) !== -1 &&
line.indexOf(stateKeywordList) !== -1 &&
@ -351,7 +343,7 @@ fs.readFile(sourceFilePath, 'utf8', function (err, data) {
// ajouter le corps complet de la section après le header
if (line.length && !isHeader) {
let cleanedLine = line.replace(/\s\s/g, ' ');
let cleanedLine = line.replace(/\s\s/g, ' ')
cleanedLine = line.replace(/ {2,}/g, ' ')
cleanedLine = cleanedLine.trim()
@ -363,17 +355,17 @@ fs.readFile(sourceFilePath, 'utf8', function (err, data) {
}
})
// ajouter la dernière tâche parsée
addAndRefreshCurrentTask();
addAndRefreshCurrentTask()
console.log(tada+ " parsing fini "+ tada)
console.log('tasks : ', tasksObjectsForJsonExport.length)
console.log(tada + ' parsing fini ' + tada)
// ranger par valeur décroissante les tags
let sorted_stats = [];
let sorted_stats = []
// rangement par valeur et par date
console.log('write file ', outputAbsolutePath, outputFileNameJson);
console.log('write file ', outputAbsolutePath, outputFileNameJson)
statistics.dates.years = sortByKey(statistics.dates.years)
statistics.dates.weeks = sortByKey(statistics.dates.weeks)
statistics.dates.months = sortByKey(statistics.dates.months)
@ -381,7 +373,6 @@ fs.readFile(sourceFilePath, 'utf8', function (err, data) {
statistics = sortByKey(statistics)
const jsonContent = {
statistics: {
lines_count: everyline.length,
@ -397,27 +388,25 @@ fs.readFile(sourceFilePath, 'utf8', function (err, data) {
tasks_list: tasksObjectsForJsonExport
}
if (writeJsonAfterParse) {
writeFileInOuputFolderFromJsonObject(outputFileNameJson, jsonContent);
writeFileInOuputFolderFromJsonObject(outputFileNameJson, jsonContent)
}
})
function lineHasKeyword (line, keyword = 'TODO') {
let isFound = (line.indexOf('* ' + keyword) !== -1)
if (isFound) {
createNewHeaderKind(keyword)
headersByKind[keyword].push(line);
headersByKind[keyword].push(line)
if (!statistics[keyword]) {
statistics[keyword] = 0
}
statistics[keyword]++
}
return isFound;
return isFound
}
function lineHasSubstring (line, keyword) {
@ -432,7 +421,7 @@ function lineHasSubstring(line, keyword) {
function createNewHeaderKind (keyword) {
if (!headersByKind[keyword]) {
headersByKind[keyword] = [];
headersByKind[keyword] = []
}
}
@ -450,14 +439,14 @@ function searchDate(line) {
let simpleDayHourSec = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.? \d{2}\:\d{2}\:\d{2}/)
if (simpleDayHourSec) {
return simpleDayHourSec;
return simpleDayHourSec
}
if (simpleDayHour) {
return simpleDayHour;
return simpleDayHour
}
if (simpleDay) {
return simpleDay;
return simpleDay
}
}
@ -468,15 +457,15 @@ function searchDate(line) {
*/
function cleanHeader (line) {
line = '' + line;
line = '' + line
stateKeywordList.forEach(keyword => {
line = line.replace(keyword, '')
})
line = line.replace(/\** /, '');
line = line.replace(/\[.*\]/g, '');
line = line.replace(/\:.*\:/g, '');
line = line.replace(' ', '');
return line.trim();
line = line.replace(/\** /, '')
line = line.replace(/\[.*\]/g, '')
line = line.replace(/\:.*\:/g, '')
line = line.replace(' ', '')
return line.trim()
}
/**
@ -487,11 +476,11 @@ function cleanHeader(line) {
function sortByKey (objectStuff) {
return Object.keys(objectStuff).sort().reduce(
(obj, key) => {
obj[key] = objectStuff[key];
return obj;
obj[key] = objectStuff[key]
return obj
},
{}
);
)
}
/**
@ -501,14 +490,14 @@ function sortByKey(objectStuff) {
*/
function sortByValue (literalobject) {
let sortable = [];
let sortable = []
for (var keyName in literalobject) {
sortable[keyName] = literalobject[keyName];
sortable[keyName] = literalobject[keyName]
}
// return literalobject
return sortable.sort(function (a, b) {
return b[1] - a[1];
});
return b[1] - a[1]
})
}
export async function writeFileInOuputFolderFromJsonObject (fileName, jsonObjectThing) {
@ -518,13 +507,13 @@ export async function writeFileInOuputFolderFromJsonObject(fileName, jsonObjectT
return await fs.writeFile(
`${outputAbsolutePath}${fileName}`,
JSON.stringify(jsonObjectThing, null, 2),
"utf8",
'utf8',
(err) => {
if (err) {
console.log(`Error writing file: ${err}`);
console.log(`Error writing file: ${err}`)
} else {
console.log(`\n ${gift} File ${fileName} is written successfully!`);
console.log(`\n ${gift} File ${fileName} is written successfully!`)
}
}
);
)
}

69
sms_to_csv.mjs
View File

@ -12,11 +12,12 @@ import { writeFileInOuputFolderFromJsonObject } from './utils.mjs'
**********************/
// const sourceFileBaseName = 'sms-20180423162531'
const sourceFileBaseName = 'sms'
const sourceFileBaseName = 'sms-20230921224605'
const sourceFileName = `${sourceFileBaseName}.xml`
const outputFileJson = `${sourceFileBaseName}.json`
const outputFileCsv = `${sourceFileBaseName}.tsv`
const outputFileCsv = `${sourceFileBaseName}.csv`
const folder_base = '/home/tykayn/Nextcloud/ressources/social sorting/'
const csv_separator = ';'
const outputFileJsonPathFull = `${folder_base}output/${outputFileJson}`
const sourceFileJson = `${folder_base}output/${outputFileJson}`
@ -54,46 +55,49 @@ function convertJsonToCsvAndPersistTSVFile (jsonFilePath, outputFileTSVName) {
console.log('convert', folder_base+'output/' + jsonFilePath)
fs.readFile(folder_base+'output/' + jsonFilePath, 'utf8', function (err, data) {
data = JSON.parse(data)
console.log('data', data.length)
if (data) {
const events = [
'amount\t' +
'content\t' +
'description\t' +
'destination\t' +
'end\t' +
'kind of activity\t' +
'person\t' +
'place\t' +
'source\t' +
'start\t' +
'unique id\t' +
'url\t'
'amount' + csv_separator + ''+
'content' + csv_separator +''+
'description' + csv_separator +
'destination' + csv_separator +
'end' + csv_separator +
'kind of activity' + csv_separator +
'person' + csv_separator +
'place' + csv_separator +
'source' + csv_separator +
'start' + csv_separator +
'unique id' + csv_separator +
'url' + csv_separator
]
data = JSON.parse(data)
// console.log('data.length', data.length)
console.log('data', Object.keys(data))
let smslist = data['elements'][1]['elements']
// console.log('data', data)
// console.log('data other ', data)
console.log('data.length', data['elements'].length)
console.log('data name', data['elements'][2]['elements'].length)
let smslist = data['elements'][2]['elements']
// let smslist = data['smses']
if (smslist) {
smslist.forEach(item => {
// convert all fields to common event description
events.push(
'\t' +
item.attributes.body.replace('\n', ' ') + '\t' +
'sms ' + item.attributes.address + ' le ' + item.attributes.readable_date + '\t' +
item.attributes.address + '\t' +
'' + '\t' +
'' + '\t' +
item.attributes.contact_name + '\t' +
'' + '\t' +
'' + '\t' +
'' + '\t' +
'' + '\t' +
'' + '\t'
''+csv_separator +
item.attributes.body + csv_separator +
'sms ' + item.attributes.address + ' le ' + item.attributes.readable_date + csv_separator +
item.attributes.address + csv_separator +
'' + csv_separator +
'' + csv_separator +
item.attributes.contact_name + csv_separator +
'' + csv_separator +
'' + csv_separator +
'' + csv_separator +
'' + csv_separator +
'' + csv_separator
)
})
// // console.log('events', events)
@ -122,9 +126,10 @@ function openSourceFile () {
fs.readFile(sourceFilePath, 'utf8', function (err, data) {
// // console.log('data', data)
var jsonConversion = convert.xml2json(data)
console.log('jsonConversion keys', Object.keys(jsonConversion))
// console.log('jsonConversion 0', Object.keys(jsonConversion['elements']))
// console.log('jsonConversion.length', parseFloat(jsonConversion.length / 1024 / 1024, 2), 'Mo')
// writeFileInOuputFolderFromJsonObject(outputFileJson, JSON.parse( jsonConversion))
writeFileInOuputFolderFromJsonObject(outputFileJson, JSON.parse(jsonConversion))
convertJsonToCsvAndPersistTSVFile(outputFileJson, outputFileCsv)

0
structure.json Normal file
View File

8
utils.mjs
View File

@ -20,11 +20,11 @@ export async function writeFileInOuputFolderFromJsonObject (fileName, jsonObject
// console.log('statistics.dates', statistics.dates)
let fullpath = `${outputAbsolutePath}${fileName}`
return await fs.writeFile(
return await fs.writeFileSync(
fullpath,
JSON.stringify(jsonObjectThing, null, 2),
// JSON.stringify(jsonObjectThing).replace('\n',''),
'utf8',
// JSON.stringify(jsonObjectThing, null, 2),
JSON.stringify(jsonObjectThing).replace('\n',''),
'UTF8',
(err) => {
if (err) {
console.log(`Error writing file: ${err}`)