org-report-stats/parse_orgmode_to_json.mjs

/**
 * convertir un fichier .org vers des données structurées en json
 * @type {*}
 */
import fs from 'node-fs';
import moment from 'moment';
import * as emoji from "node-emoji";


/**********************
 * initialize configs
 **********************/

const sourceFileName = 'all_tasks.org'
const sourceFilePath = './sources/' + sourceFileName;
const outputAbsolutePath = '~/Nextcloud/ressources/social sorting/output/';
const outputFileNameJson = 'export_' + sourceFileName + '_parsed.json';

let headers = []
let tasksObjectsForJsonExport = []
let headersByKind = {}
let writeJsonAfterParse = false;
writeJsonAfterParse = true;
moment.locale('FR')

const tada = emoji.get("tada");
const gift = emoji.get("gift");

/**************************************************************
 * fetch the source orgmode file to read its contents
 *************************************************************/

console.log('---------- parse some org file', sourceFilePath)
if (!sourceFilePath) {
    console.error('pas de fichier à ouvrir')
}
fs.stat(sourceFilePath, function (err, stat) {
    if (err == null) {
        console.log(`File ${sourceFilePath} exists`);

    } else if (err.code === 'ENOENT') {
        // file does not exist
        console.error(`le fichier ${sourceFilePath} est introuvable. Impossible d en extraire des infos.`, err);
    } else {
        console.log('Some other error: ', err.code);
    }
});

/**********************
 * search elements
 *********************/
let stateKeywordList = ['SOMEDAY', 'NEXT', 'TODO', 'CANCELLED', 'DONE', 'WAITING'];
let dateKeywordList = ['CREATED', 'SCHEDULED', 'DEADLINE', 'CLOSED', 'Refiled'];
let sectionKeywordList = ['PROPERTIES', 'LOGBOOK', 'END'];

let propertiesSection = {} // TODO properties listing
let logBookSection = {} // TODO logbook listing

let statistics = {
    tags: {},
    words: {},
    dates: {
        havingDate: 0,
        havingNoDate: 0,
        oldEst: 0,
        mostRecent: 0,
        years: {},
        weeks: {},
        months: {},
        days: {}
    }
}

let headerKeywordSearch = '[' + stateKeywordList.join('|') + ']'
/**
 * task object example
 * @type {{level: string, header: string, dates: {CREATED: string, DONE: string, REFILED: string}, state: string, content: string, properties: {}, tags: [], tagsInherited: []}}
 */
let task = {
    header: "",
    level: "",
    corpus: "",
    state: "",
    tags: [],
    children: [], // TODO list children tasks with a reference to the parent when level is superior to previous task
    tagsInherited: [], // TODO inherit tags
    dates: {
        havingDate: 0,
        havingNoDate: 0,
        oldEst: 0,
        mostRecent: 0,
        years: {},
        weeks: {},
        months: {},
        days: {},
    },
    logbook: {},
    properties: {},
}

let isHeader = false;
let isProperty = false;
let isLogbook = false;
let isFirst = true;

// init first task object as empty clone
let currentTask = Object.create(task);

/**
 * add to tasks to export and refresh current task
 */
function addAndRefreshCurrentTask() {

    makeWordsStatistics(currentTask.header.trim())
    makeWordsStatistics(currentTask.corpus.trim())

    tasksObjectsForJsonExport.push(currentTask)
    // réinitialisation de tâche pour remplir de nouveau
    currentTask = Object.create(task);
    currentTask.dates = {};
    currentTask.tags = [];
}

function makeWordsStatistics(sentence) {
    let split = sentence.split(' ');
    if (split && split.length) {

        split.forEach(word => {
            if (!statistics.words[word]) {
                statistics.words[word] = 0
            }
            statistics.words[word]++
        })
    }
}

const dateStats = {
    created: 0,
    refiled: 0,
    closed: 0,
    cancelled: 0,
    scheduled: 0,
    deadline: 0,
    tasks_done: []
}

function fillPeriodTime(periodStat, keyword){

    if (keyword === 'CLOSED') {
        periodStat.closed++;
    }
    if (keyword === 'CREATED') {
        periodStat.created++;
    }
    if (keyword === 'Refiled') {
        periodStat.refiled++;
    }
    if (keyword === 'CANCELLED') {
        periodStat.cancelled++;
    }
    if (keyword === 'SCHEDULED') {
        periodStat.scheduled++;
    }
    if (keyword === 'DEADLINE') {
        periodStat.deadline++;
    }
}

/**
 * pour chaque période de temps, compter les tâches créées et fermées
 * @param keyword
 * @param dateFoundElement
 */
function statisticDateFill(keyword, dateFoundElement, header) {

    // décompte par années

    let convertedDate = new Date(dateFoundElement)
    let yearOfDate = convertedDate.getFullYear()
    let monthOfDate = yearOfDate + '-' + convertedDate.getMonth()
    // add zeros
    let convertedWeek = moment(convertedDate).week() < 10 ? '0' + moment(convertedDate).week() : moment(convertedDate).week()
    let weekOfDate = yearOfDate + '-' + convertedWeek

    let convertedMonth = convertedDate.getMonth() < 10 ? '0' + convertedDate.getMonth() : convertedDate.getMonth()
    let convertedDay = convertedDate.getDay() < 10 ? '0' + convertedDate.getDay() : convertedDate.getDay()
    let dayOfDate = convertedDate.getFullYear() + '-' + convertedMonth + '-' + convertedDay

    if (!statistics.dates.years[yearOfDate]) {
        statistics.dates.years[yearOfDate] = Object.create(dateStats)
    }
    fillPeriodTime(statistics.dates.years[yearOfDate], keyword)

    // par année-semaine
    if (!statistics.dates.weeks[weekOfDate]) {
        statistics.dates.weeks[weekOfDate] = Object.create(dateStats)
    }
    fillPeriodTime(statistics.dates.weeks[weekOfDate], keyword)
    statistics.dates.weeks[weekOfDate].tasks_done.push(header)
    // décompte par mois
    if (!statistics.dates.months[monthOfDate]) {
        statistics.dates.months[monthOfDate] = Object.create(dateStats)
    }
    fillPeriodTime(statistics.dates.months[monthOfDate], keyword)

    // décompte par jours

    if (!statistics.dates.days[dayOfDate]) {
        statistics.dates.days[dayOfDate] = Object.create(dateStats)
    }
    fillPeriodTime(statistics.dates.days[dayOfDate], keyword)
    statistics.dates.days[dayOfDate].tasks_done.push(header)
}

function findOldestDate(currentDate) {
    // trouver la plus ancienne date


    if (!statistics.dates.oldEst) {
        statistics.dates.oldEst = currentDate;
    } else {
        var beginningTime = moment(statistics.dates.oldEst);
        var endTime = moment(currentDate);
        if (!beginningTime.isBefore(endTime)) {
            statistics.dates.oldEst = currentDate;
        }
    }
}

/**********************
 * loop to parse all
 *********************/
fs.readFile(sourceFilePath, 'utf8', function (err, data) {


    if (err) {
        return console.log(err);
    }
    console.log(" parsing...")
    // parcourir chaque ligne du fichier org
    let everyline = data.split('\n');

    // trouver les entêtes toutes les lignes qui commencent par * et espace.

    everyline.forEach((line) => {


        // gérer la création d'objets définissant les tâches et leurs propriétés
        if (line.match(/^\*+? /)) {
            // add last task to export list
            if (!isFirst) {

                addAndRefreshCurrentTask();
            } else {
                isFirst = false;
            }

            isHeader = true;
            // compter les étoiles pour trouver le niveau du header
            if (line.match(/\*/g)) {

                let match = line.match(/\*/g);
                currentTask.level = match.length
            }

            // create a new task


            headers.push(cleanHeader(line))
            currentTask.header = cleanHeader(line);
            stateKeywordList.forEach(keyword => {
                let keywordIsFound = lineHasKeyword(line, keyword)

                if (keywordIsFound) {
                    currentTask.state = keyword
                }
            })


            // trouver les tags
            let tagsList = line.match(/\:([\w\_]*)\:/g)
            if (tagsList) {
                tagsList = tagsList[0];
                let tagList = tagsList.split(':');
                if (tagList.length) {

                    tagList.forEach(tag => {
                        if (tag.length > 1) {

                            if (!statistics.tags[tag]) {
                                statistics.tags[tag] = 0
                            }
                            statistics.tags[tag]++

                            currentTask.tags.push(tag)
                        }
                    })
                }
            }


            // ------------- fin des recherches dans la ligne de Header -------------
        } else {
            isHeader = false;
        }
        // examen des lignes de corps de tâche, ou de corps de section suite au header.
        // classer les dates de création, cloture, et de logbook
        let dateFound = searchDate(line)
        if (dateFound) {
            /**
             * we have found a date in the current line
             */

            statistics.dates.havingDate += 1;

            dateKeywordList.forEach(keyword => {
                if (lineHasSubstring(line, keyword)) {
                    if (!currentTask.dates[keyword]) {
                        currentTask.dates[keyword] = '';
                    }

                    let convertedDate = dateFound[0].substring(0, 10);

                    if (dateFound[0].length == 15) {
                        // sans heure: "2022-12-21 mer."
                        convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd')

                    } else if (dateFound[0].length == 21) {
                        // avec heure: "2022-11-01 mar. 00:44"
                        convertedDate = moment(dateFound[0], 'YYYY-MM-DD ddd HH:mm')
                    }
                    let formattedDate = moment(convertedDate).format()
                    console.log('currentTask.header', currentTask.header)
                    statisticDateFill(keyword, convertedDate, currentTask.header)
                    findOldestDate(convertedDate)
                    currentTask.dates[keyword] = formattedDate;


                } else {
                    // console.log('keyword', keyword)
                }
            })
        } else {
            statistics.dates.havingNoDate += 1;
            if (
                line.indexOf(dateKeywordList) !== -1 &&
                line.indexOf(stateKeywordList) !== -1 &&
                line.indexOf(sectionKeywordList) !== -1
            ) {

                // ajouter le corps complet de la section après le header
                if (line.length && !isHeader) {

                    let cleanedLine = line.replace(/\s\s/g, ' ');
                    cleanedLine = line.replace(/ {2,}/g, ' ')

                    cleanedLine = cleanedLine.trim()

                    currentTask.corpus += `${cleanedLine}
`
                }
            }
        }
    })


    // ajouter la dernière tâche parsée
    addAndRefreshCurrentTask();

    console.log(tada+ " parsing fini "+ tada)

    // ranger par valeur décroissante les tags

    let sorted_stats = [];
    // rangement par valeur et par date
    console.log('write file ', outputAbsolutePath, outputFileNameJson);
    statistics.dates.years = sortByKey(statistics.dates.years)
    statistics.dates.weeks = sortByKey(statistics.dates.weeks)
    statistics.dates.months = sortByKey(statistics.dates.months)
    statistics.dates.days = sortByKey(statistics.dates.days)

    statistics = sortByKey(statistics)


    const jsonContent = {
        statistics: {
            lines_count: everyline.length,
            headers_count: headers.length,
            statistics
        },
        meta_data: {
            author: '@tykayn@mastodon.Cipherbliss.com',
            generated_at: new Date(),
            generated_from_file: sourceFilePath + sourceFileName,
            sources: 'https://forge.chapril.org/tykayn/org-report-stats.git'
        },
        tasks_list: tasksObjectsForJsonExport
    }


    if (writeJsonAfterParse) {

        writeFileInOuputFolderFromJsonObject(outputFileNameJson, jsonContent);
    }

})


function lineHasKeyword(line, keyword = 'TODO') {

    let isFound = (line.indexOf('* ' + keyword) !== -1)
    if (isFound) {
        createNewHeaderKind(keyword)
        headersByKind[keyword].push(line);
        if (!statistics[keyword]) {
            statistics[keyword] = 0
        }
        statistics[keyword]++
    }
    return isFound;
}

function lineHasSubstring(line, keyword) {
    let isFound = (line.indexOf(keyword) !== -1)
    if (!statistics[keyword]) {
        statistics[keyword] = 0
    }
    statistics[keyword]++

    return isFound
}

function createNewHeaderKind(keyword) {
    if (!headersByKind[keyword]) {
        headersByKind[keyword] = [];
    }
}

/**
 * chercher des dates et heures au format
 * YYYY-MM-DD HH:II:SS
 *
 * @param line
 * @returns {*}
 */
function searchDate(line) {
    // return line.match(/[(\d{4}\-\d{2}\-\d{2} ?\d{2}?\:?\d{2}?\:?\d{2}?)(\d{4}\-\d{2}\-\d{2})]/)
    let simpleDay = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.?/)
    let simpleDayHour = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.? \d{2}\:\d{2}/)
    let simpleDayHourSec = line.match(/\d{4}\-\d{2}\-\d{2} \w{3}?\.? \d{2}\:\d{2}\:\d{2}/)

    if (simpleDayHourSec) {
        return simpleDayHourSec;
    }

    if (simpleDayHour) {
        return simpleDayHour;
    }
    if (simpleDay) {
        return simpleDay;
    }

}

/**
 * get the cleaned content of the header
 * @param line
 */
function cleanHeader(line) {

    line = '' + line;
    stateKeywordList.forEach(keyword => {
        line = line.replace(keyword, '')
    })
    line = line.replace(/\** /, '');
    line = line.replace(/\[.*\]/g, '');
    line = line.replace(/\:.*\:/g, '');
    line = line.replace('  ', '');
    return line.trim();
}

/**
 * ranger un objet littéral par ordre alphabétique de ses clés
 * @param objectStuff
 * @returns {{}}
 */
function sortByKey(objectStuff) {
    return Object.keys(objectStuff).sort().reduce(
        (obj, key) => {
            obj[key] = objectStuff[key];
            return obj;
        },
        {}
    );
}

/**
 * ranger un objet littéral selon la valeur décroissante de ses paires
 * @param literalobject
 * @returns {any}
 */
function sortByValue(literalobject) {

    let sortable = [];
    for (var keyName in literalobject) {
        sortable[keyName] = literalobject[keyName];
    }
    // return literalobject
    return sortable.sort(function (a, b) {
        return b[1] - a[1];
    });
}

export async function writeFileInOuputFolderFromJsonObject(fileName, jsonObjectThing) {

    // console.log('statistics.dates', statistics.dates)

    return await fs.writeFile(
        `${outputAbsolutePath}${fileName}`,
        JSON.stringify(jsonObjectThing, null,2),
        "utf8",
        (err) => {
            if (err) {
                console.log(`Error writing file: ${err}`);
            } else {
                console.log(`\n ${gift} File ${fileName} is written successfully!`);
            }
        }
    );
}