Commit 3bda33cc authored by ty kayn's avatar ty kayn

📖 some more doc

parent 2d1c8398
......@@ -20,6 +20,12 @@ and you will see the result in html.
## What statistics does this provide
For the moment:
* who you talked to the most, with counters. This uses the object.cc field of a toot.
* link to search on duckduckgo for hashtags and usernames
* what are your most used hashtags
* you can filter:
* only toots with medias
* change the order of filter
* min length of toot content
* what toots are containing at least a certain number of characters. You can use this to retrieve some long posts you made, to post them on a worthy place where you would develop your subject deeper. Somewhere like on your personal website, thats an easy way to find your longests posts. But be careful, it could also show how much you talk with trolls :D
## configuration
......
......@@ -2,10 +2,11 @@ exports.name = 'masto_conversion';
const fs = require('fs');
class Conversion {
hello(){
console.log('hello from conversion')
hello() {
console.log('hello from conversion');
}
likes(){
likes() {
// read file likes
fs.readFile('source_data/likes.json',
// callback function that is called when reading file is done
......@@ -19,68 +20,118 @@ class Conversion {
});
}
filterToots(toots,options){
let minchartoots ;
filterToots(toots, options) {
let minchartoots;
if(options.filterBiggerTottsBeforeSlicing){
if (options.filterBiggerTottsBeforeSlicing) {
minchartoots = toots.filter(item => {
return item['object'].content && item['object'].content.length > options.min_length;
});
minchartoots = minchartoots.slice(0, options.max_toots);
}else{
} else {
const slice = toots.slice(0, options.max_toots);
minchartoots = slice.filter(item => {
return item['object'].content && item['object'].content.length > options.min_length;
});
}
return minchartoots
minchartoots.forEach(toot => {
toot = this.findMediaUrl(toot);
toot = this.removeLastChars(toot);
return toot;
});
return minchartoots;
}
makeStatsForToots(tootArray){
let stats = { recievers: {},
hashtags : {}};
// make statistics on who do we talk to, based on the cc field
tootArray.forEach(elem => {
// stats on hashtags
if (elem['object'].tag) {
elem['object'].tag.forEach(tag => {
if(tag.type === 'Hashtag'){
if(!stats.hashtags[tag.name]){
stats.hashtags[tag.name] = {
name : tag.name,
counter : 0,
}
}
stats.hashtags[tag.name].counter++;
}
})
}
// stats on recievers of toots
if (elem['object'].cc) {
elem['object'].cc.forEach(copyFolk => {
if (!stats.recievers[copyFolk]) {
stats.recievers[copyFolk] = {
name : copyFolk,
//
/**
*
* @param toot
* @returns {*}
*/
findMediaUrl(toot) {
/**
* goal:
* https://mastodon.cipherbliss.com/system/media_attachments/files/000/858/113/original/74b370672892f884.jpg?1566230144
*
* input data:
* "attributedTo":"https://mastodon.cipherbliss.com/users/tykayn",
*
* "attachment":[{"type":"Document",
* "mediaType":"image/png","url":"media_attachments/files/000/872/910/original/c82b422f302b8ec9.png",
* "name":null,
* "blurhash":"UnSOjgo~ysWAVYWBkWjXu5axVrjckqoze?Rk"}],
*
* we use the attribuedTo property to find instance, and map the url to the instance url, and add this property to the attachment
*/
if (toot['object'].attributedTo) {
let splitted = toot['object'].attributedTo.split('/');
let instanceUrl = splitted[2];
toot.instanceUrl = 'https://' + instanceUrl;
toot.attachment = toot['object'].attachment.map(att => {
att.href = toot.instanceUrl + '/system/' + att.url;
return att;
});
}
return toot;
}
removeLastChars(toot) {
toot['object'].content = toot['object'].content.trim();
return toot;
}
makeStatsForToots(tootArray) {
let stats = {
recievers: {},
hashtags : {},
};
// make statistics on who do we talk to, based on the cc field
tootArray.forEach(elem => {
// stats on hashtags
if (elem['object'].tag) {
elem['object'].tag.forEach(tag => {
if (tag.type === 'Hashtag') {
if (!stats.hashtags[tag.name]) {
stats.hashtags[tag.name] = {
name : tag.name,
href : tag.href,
counter: 0,
counterContentLength: 0,
};
}
stats.recievers[copyFolk].counter++;
stats.recievers[copyFolk].counterContentLength += elem['object'].content.length;
});
}
});
stats.hashtags[tag.name].counter++;
}
});
}
// stats on recievers of toots
if (elem['object'].cc) {
elem['object'].cc.forEach(copyFolk => {
if (!stats.recievers[copyFolk]) {
stats.recievers[copyFolk] = {
user : this.urlToUser(copyFolk),
name : copyFolk,
counter : 0,
counterContentLength: 0,
};
}
stats.recievers[copyFolk].counter++;
stats.recievers[copyFolk].counterContentLength += elem['object'].content.length;
});
}
});
console.log('stats.hashtags', stats.hashtags);
stats = {
recievers : this.sortTootsByLength(stats.recievers),
hashtags : this.sortTootsByLength(stats.hashtags),
};
return stats ;
console.log('stats.hashtags', stats.hashtags[0]);
stats = {
recievers: this.sortTootsByLength(stats.recievers),
hashtags : this.sortTootsByLength(stats.hashtags),
};
return stats;
}
sortTootsByLength(stats){
sortTootsByLength(stats) {
const statKeys = Object.keys(stats);
const arrayToSort = [];
statKeys.forEach(elem => {
......@@ -88,10 +139,28 @@ class Conversion {
stats[elem],
);
});
arrayToSort.sort( (a,b)=>{
return b.counter - a.counter
arrayToSort.sort((a, b) => {
return b.counter - a.counter;
});
return arrayToSort;
}
urlToUser(url) {
let sliceOfSlashes = url.split('/');
let userObject = {
url : url,
username: sliceOfSlashes[sliceOfSlashes.length - 1],
};
return userObject;
}
filterOnlyTootsWithMedias(tootList) {
console.log('filterOnlyTootsWithMedias')
return tootList.filter(toot => {
return toot['object'].attachment && toot['object'].attachment.length
});
}
}
exports.conversion = new Conversion();
......@@ -10,58 +10,68 @@ var fs = require('fs');
var listenPort = 8088;
var jsonParsedLikes, jsonParsedOutbox;
// const min_length = 1050; // filter only long toots
const min_length = 300; // filter only long toots
const max_toots = 500; // filter only long toots
const min_length = 1; // filter only long toots
const max_toots = 10; // filter only long toots
const filterBiggerTottsBeforeSlicing = false; // filter only long toots
const filterOnlyHavingMedias = true; // filter only toots having medias
const writeStatsJson = false; // filter only toots having medias
const TemplateVars = {
pageTitle : 'Mastodon export converter to HTML',
likes : jsonParsedLikes,
outbox : jsonParsedOutbox,
outboxStatistics: {},
outbox_all : jsonParsedOutbox,
min_length ,
max_toots ,
min_length,
max_toots,
filterOnlyHavingMedias,
filterBiggerTottsBeforeSlicing,
writeStatsJson,
};
const masto_converter = require('./conversion');
console.log('masto_converter', masto_converter);
masto_converter.conversion.hello();
jsonParsedLikes = masto_converter.conversion.likes();
fs.readFile('source_data/outbox.json',
// callback function that is called when reading file is done
function (err, data) {
let minchartoots ;
let toots;
// parse json
jsonParsedOutbox = JSON.parse(data);
toots = jsonParsedOutbox.orderedItems;
// access elements
console.log('outbox toots length', jsonParsedOutbox.orderedItems.length);
console.log('outbox toots length', toots.length);
TemplateVars.outboxTotalLength = jsonParsedOutbox.orderedItems.length;
minchartoots = masto_converter.conversion.filterToots(jsonParsedOutbox.orderedItems, TemplateVars)
TemplateVars.outboxTotalLength = toots.length;
toots = jsonParsedOutbox.orderedItems;
if (filterOnlyHavingMedias) {
toots = masto_converter.conversion.filterOnlyTootsWithMedias(toots);
console.log('toots.length only attachements', toots.length );
}
toots = masto_converter.conversion.filterToots(toots, TemplateVars);
console.log('min_chars', min_length);
console.log('toots min char corresponding', minchartoots.length);
TemplateVars.outbox = minchartoots;
TemplateVars.outboxStatistics = masto_converter.conversion.makeStatsForToots(minchartoots);
console.log('toots min char corresponding', toots.length);
TemplateVars.outbox = toots;
TemplateVars.outboxStatistics = masto_converter.conversion.makeStatsForToots(toots);
const example = TemplateVars.outbox[1]['object'];
TemplateVars.example = example;
console.log('example', example)
fs.writeFile('output/statistics.json', JSON.stringify(TemplateVars.outboxStatistics), errfileHandler );
if (writeStatsJson) {
fs.writeFile('output/statistics.json', JSON.stringify(TemplateVars.outboxStatistics), errfileHandler);
}
});
const errfileHandler = (err) => {
if (err)
console.log(err);
else {
console.log("File statistics written successfully\n");
console.log('File statistics written successfully\n');
}
}
};
app.use(express.static('public'));
app.set('view engine', 'pug');
......@@ -70,7 +80,7 @@ app.get('/', (req, res) => {
const html = pug.render('index.pug', TemplateVars);
fs.writeFile('output/my_toots.html', html, errfileHandler );
fs.writeFile('output/my_toots.html', html, errfileHandler);
res.render('index.pug', TemplateVars);
......
body{
background: #222;
body {
background: #292a2a;
color: white;
padding: 1em 2em;
}
a{
color: #5561ff;
a {
color: #00a7d1;
}
.status{
background: #111;
.status {
background: #606984;
padding: 0.5em;
border-radius: 0.25em;
border-bottom: 1px solid #42495c;
}
.date-published{
font-size: 0.5em;
.published {
width: 70ch;
}
.status .published{
padding-left: 2em;
.date-published {
font-size: 0.5em;
}
.stats{
.stats {
max-height: 15em;
overflow: auto;
}
.media-gallery {
}
.media-media-displayed {
min-height: 500px;
margin-right: 1em;
}
.hidden {
display: none;
}
......@@ -13,13 +13,17 @@ html(lang="en")
table
thead
tr
th= "search"
th= "hashtag"
th= "occurences"
tbody
each hashtag in outboxStatistics.hashtags
tr
td.search
a(href= 'https://duckduckgo.com/?q='+hashtag.name)
='search'
td.name
a(href=hashtag.name)=hashtag.name
a(href=hashtag.href)=hashtag.name
td.counter=hashtag.counter
td.counter=hashtag.counterContentLength
......@@ -28,14 +32,18 @@ html(lang="en")
table
thead
tr
th= "search"
th= "name"
th= "times"
th= "toots lengh sum"
tbody
each someone in outboxStatistics.recievers
tr
td.search
a(href= 'https://duckduckgo.com/?q='+someone.user.username)
='search'
td.name
a(href=someone.name)=someone.name
a(href=someone.name)=someone.user.username
td.counter=someone.counter
td.counter=someone.counterContentLength
h2 #{outbox.length} of Messages #{outboxTotalLength} in your outbox.First #{max_toots} toots, filtered by a minimal length of #{min_length} characters of content.
......@@ -47,4 +55,21 @@ html(lang="en")
div.status.status-public
a(href=oredredItem['object'].url)="see"
div.date-published=oredredItem['object'].published
blockquote.published(unescaped!=oredredItem['object'].content)
blockquote.published(escaped!=oredredItem['object'].content)
if oredredItem['object'].attachment && oredredItem['object'].attachment.length
each media in oredredItem['object'].attachment
div.media-gallery
div.counter.hidden #{oredredItem['object'].attachment.length}
if(media.mediaType.search('video'))
figure.media-media-displayed.media-image
a(href=media.href)
video(src=media.href)
span Media vidéo
elseif(media.mediaType.search('audio'))
span Media audio
elseif(media.mediaType.search('image'))
figure.media-media-displayed.media-image
a(href=media.href)
img(src=media.href,alt=media.href)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment