From: Zachary Vance Date: Wed, 31 Jul 2024 23:18:24 +0000 (-0400) Subject: Gut HTML extraction, HTML->markdown conversion, side-by-side view, and visual regress... X-Git-Url: https://git.za3k.com/?a=commitdiff_plain;h=e707a9c65a75d2d042d275688053ca13fae1ddba;p=blog.git Gut HTML extraction, HTML->markdown conversion, side-by-side view, and visual regression tool --- diff --git a/.gitignore b/.gitignore index c1e054c..3dc3a52 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /__pycache__ /node_modules /visual-diff.csv +/screenshots diff --git a/blog b/blog index f40e2bc..8b8ff0c 100755 --- a/blog +++ b/blog @@ -188,7 +188,6 @@ class Templatable(PseudoMap): html = mustache.render(template, context, warn=True) if replace_links: html = blog.replace_links(source, html) - html = blog.small_corrections(html) return html def content(self): @@ -226,12 +225,10 @@ class Static(Templatable): return self.path.relative_to(self.blog.static_dir) class Post(Templatable): - def __init__(self, parsed, blog, parsed_md): + def __init__(self, parsed, blog, comments): super().__init__(blog) - self.post, self.comments = parsed.pop("content").split("\n") - self.post = self.post.replace("","") + self.md, self.comments = parsed.pop("content"), comments self.main_display = True - self.md = parsed_md["content"] for k, v in parsed.items(): if k in {"tags", "author", "categories"}: k = "_" + k @@ -241,59 +238,11 @@ class Post(Templatable): # TODO: Add an 'above the fold' part, maybe @property - def md_html(self): + def post(self): return '

{}

'.format(markdown2html(self.md)) - @property - def output_path_html(self): - output_path_template = self.blog["post_html_destination"] - return Path(mustache.render(output_path_template, self.context)) - - @property - def output_path_md(self): - output_path_template = self.blog["post_md_destination"] - return Path(mustache.render(output_path_template, self.context)) - - @property - def url_md(self): - return self.blog.web_root + "/" + str(self.output_path_md.relative_to(self.blog.destination)) - - @property - def url_html(self): - return self.blog.web_root + "/" + str(self.output_path_html.relative_to(self.blog.destination)) - def content_combined(self): - return self.render_template(self.blog, "postcombined", self.context, replace_links=False).encode("utf8") - - def content_md(self): - content = self.render_template(self.blog, self.type, collections.ChainMap({ - "post": self.md_html - }, self.context)) - return self.render_template(self.blog, "layout", collections.ChainMap({ - "content": content, - }, self, self.blog)).encode("utf8") - - def output(self): - output = self.content() - if self.blog.reload: - output += RELOAD_HTML - self.output_path_html.parent.mkdir(parents=True, exist_ok=True) - with open(self.output_path_html, "wb") as f: - f.write(output) - - output = self.content_md() - if self.blog.reload: - output += RELOAD_HTML - self.output_path_md.parent.mkdir(parents=True, exist_ok=True) - with open(self.output_path_md, "wb") as f: - f.write(output) - - output = self.content_combined() - if self.blog.reload: - output += RELOAD_HTML - self.output_path.parent.mkdir(parents=True, exist_ok=True) - with open(self.output_path, "wb") as f: - f.write(output) + return self.render_template(self.blog, "post", self.context, replace_links=False).encode("utf8") @property def date_rfc822(self): @@ -434,24 +383,23 @@ class Blog(PseudoMap): link_regex = '(?

Opening up ComputerCraft to the world using the http commands was an amazing idea from the developers!

-Reply

- - \ No newline at end of file + diff --git a/posts-comments/ddos.html b/comments/ddos.html similarity index 73% rename from posts-comments/ddos.html rename to comments/ddos.html index 32e9c63..9d22dac 100644 --- a/posts-comments/ddos.html +++ b/comments/ddos.html @@ -10,8 +10,5 @@

Perhaps look into http://trilema.com/2015/mika-epstein-aka-ipstenu-is-a-thoroughly-clueless-poser/

-Reply

- - \ No newline at end of file + diff --git a/posts-comments/diy-keyboards-and-how-keyboards-work.html b/comments/diy-keyboards-and-how-keyboards-work.html similarity index 73% rename from posts-comments/diy-keyboards-and-how-keyboards-work.html rename to comments/diy-keyboards-and-how-keyboards-work.html index 595495f..d5b8cba 100644 --- a/posts-comments/diy-keyboards-and-how-keyboards-work.html +++ b/comments/diy-keyboards-and-how-keyboards-work.html @@ -11,8 +11,5 @@

âAnd better keyboards can detect multiple keys being pressed at once (N-key rollover), which I think they do by having a completely separate wire to each key.â

You can keep a matrix arrangement and have N-key rollover by putting a diode in series with every switch

-Reply

- - \ No newline at end of file + diff --git a/posts-comments/hack-a-day-day-04-lashed-table.html b/comments/hack-a-day-day-04-lashed-table.html similarity index 68% rename from posts-comments/hack-a-day-day-04-lashed-table.html rename to comments/hack-a-day-day-04-lashed-table.html index a255ed8..8f4198c 100644 --- a/posts-comments/hack-a-day-day-04-lashed-table.html +++ b/comments/hack-a-day-day-04-lashed-table.html @@ -10,8 +10,5 @@

Cool!

-Reply

- - \ No newline at end of file + diff --git a/posts-comments/installing-email-with-postfix-and-dovecot.html b/comments/installing-email-with-postfix-and-dovecot.html similarity index 72% rename from posts-comments/installing-email-with-postfix-and-dovecot.html rename to comments/installing-email-with-postfix-and-dovecot.html index f705bec..b6b71a2 100644 --- a/posts-comments/installing-email-with-postfix-and-dovecot.html +++ b/comments/installing-email-with-postfix-and-dovecot.html @@ -12,9 +12,6 @@

Hello,

Can you implement quota support from postgressql?

-Reply

@@ -26,9 +23,6 @@

Not sure if this is âwill you doâ or âis it possibleâ. I will not do anything to set up quotaâI have one user (me) so it doesnât make sense. I encourage you to link if you do.

-
-Reply
-

@@ -43,9 +37,6 @@

In the âuseradd -d -M -d /var/mail/vmail âshell=/usr/bin/nologin -u 5000 -g vmail vmailâ I think the first â-dâ shouldnât be there

Thanks for the post

-Reply

@@ -57,9 +48,6 @@

Thanks for the corrections, glad you found it useful.

-
-Reply
-

@@ -75,9 +63,6 @@ CREATE USER postfix PASSWORD âXXXâ;
CREATE USER dovecot PASSWORD âXXXâ;

-Reply

@@ -92,9 +77,6 @@ CREATE USER dovecot PASSWORD âXXXâ;

-Reply

@@ -107,9 +89,6 @@ CREATE USER dovecot PASSWORD âXXXâ;

A little sloppy â you should have simply shown the entire contents of each file one by one, with descriptions in comments or whatever, instead of breaking them up into snippets â and there are a few errors and omissions (I canât recall which ones now)
However, using this guide and some googling, I was able to get a mailserver working â which is more than I can say for any other guide; so, thanks.

-Reply

@@ -121,9 +100,6 @@ However, using this guide and some googling, I was able to get a mailserver work

A guide that could have been great but tripped at the finish line. A lot of errors in SQL and elsewhere along with step reorganization.

-Reply

@@ -136,10 +112,7 @@ However, using this guide and some googling, I was able to get a mailserver work
If you have any specific errors you can mention, Iâd be happy to fix stuff.

But yeah, thatâs the problem with writing a guide after you do something instead of during/before.

-
-Reply
-

- \ No newline at end of file + diff --git a/posts-comments/introducing-the-zorchpad-display-demo.html b/comments/introducing-the-zorchpad-display-demo.html similarity index 83% rename from posts-comments/introducing-the-zorchpad-display-demo.html rename to comments/introducing-the-zorchpad-display-demo.html index 20c4b08..bbb3dcf 100644 --- a/posts-comments/introducing-the-zorchpad-display-demo.html +++ b/comments/introducing-the-zorchpad-display-demo.html @@ -16,9 +16,6 @@

One thing that struck me is that a lot of people might want a Xorchpad to stick INSIDE a larger system (out in nature? as part of a science package?) that might install a rugged solar panel and battery. Then a small USB-C cord might provide a Xorchpad inside the unit with power, and want some intelligence to come out of the same cord, in exchange?

Iâm not sure if a USB-C is consistent with the vision, however. Maybe the âminimum powerâ for that is far above the âmaximim powerâ that would not blow the Sourcepadâs circuits?

-Reply

@@ -34,10 +31,7 @@
The Zorchpad could easily be powered by any standard cable, yes. But if you need a larger system to power it, youâre missing the point.

âBlowing circuitsâ is not a correct intuition for designing low-power electronics. Go learn about Ohmâs Law!

-
-Reply
-

- \ No newline at end of file + diff --git a/posts-comments/making-a-hardware-random-number-generator.html b/comments/making-a-hardware-random-number-generator.html similarity index 72% rename from posts-comments/making-a-hardware-random-number-generator.html rename to comments/making-a-hardware-random-number-generator.html index b3210ed..e9cbf44 100644 --- a/posts-comments/making-a-hardware-random-number-generator.html +++ b/comments/making-a-hardware-random-number-generator.html @@ -10,9 +10,6 @@

Hello, I think you are truely right about the TRNG. I am a researcher who specializes in designing TRNG. And now I have a TRNG chip with PCB support USB2.0. And the speed can be up to 30Mbps. It can pass NIST 800-22 and 800-90B. Are you interested in that? Looking forward to your reply!!

-Reply

@@ -26,8 +23,5 @@

Is your TRNG open-source?

Where do you research?

-Reply

- \ No newline at end of file + diff --git a/posts-comments/making-my-finances-public.html b/comments/making-my-finances-public.html similarity index 70% rename from posts-comments/making-my-finances-public.html rename to comments/making-my-finances-public.html index 740f4b9..04a09d4 100644 --- a/posts-comments/making-my-finances-public.html +++ b/comments/making-my-finances-public.html @@ -10,9 +10,6 @@

is offline

-Reply

@@ -24,10 +21,7 @@

Yeah, my export broke a while back. This should be fixed indefinitely, but covers 2011-2016 only now.

-
-Reply
-

- \ No newline at end of file + diff --git a/posts-comments/printing-on-the-brother-hl-2270dw-printer-using-a-raspberry-pi.html b/comments/printing-on-the-brother-hl-2270dw-printer-using-a-raspberry-pi.html similarity index 72% rename from posts-comments/printing-on-the-brother-hl-2270dw-printer-using-a-raspberry-pi.html rename to comments/printing-on-the-brother-hl-2270dw-printer-using-a-raspberry-pi.html index 6b87121..bf91c20 100644 --- a/posts-comments/printing-on-the-brother-hl-2270dw-printer-using-a-raspberry-pi.html +++ b/comments/printing-on-the-brother-hl-2270dw-printer-using-a-raspberry-pi.html @@ -11,9 +11,6 @@

There appears to be a typo in step 5A. The page currently reads âlpinfo -mâ but I believe should be âlpinfo -vâ. Per the man page, the m flag lists drivers and the v flag lists devices.

The USB device can be found in the -v output for step 5A but the driver can be found in the -m output for step 5B.

-Reply

@@ -26,9 +23,6 @@

Thank you so very much for this. Worked like a charm.

Any tips on how to print over the network?

-Reply

@@ -40,9 +34,6 @@

Awesome post. Joel is supposedly correct. It should have been âlpinfo -vâ in the step 5A.

-Reply

@@ -55,9 +46,6 @@

Corrected âlpinfo -vâ, thanks.
No clue how to print over the network, sorry. Thatâs actually why I set up a raspberry pi to connect to the printer insteadâitâs my wifi interface.

-Reply

@@ -72,9 +60,6 @@ No clue how to print over the network, sorry. Thatâs actually why I set up a r

For printing over the network, I used the socket::9100 setting. No idea the command line knobs or dials, just did it through the cups web page.

Another reason for me to do this is now my little rasperry pi zero w shows my printer as an AirPrint printer, so now I can print from my iDevices. Brother has their own app for this printer, but I was annoyed having to install a special app for this. My HL2270DW was made without AirPrint. Itâs a fine little machine and donât want to throw it out.

-Reply

@@ -87,9 +72,6 @@ No clue how to print over the network, sorry. Thatâs actually why I set up a r
How did you install from source? and how did you do the port forwarding across via NAT to another internal network? Iâm new to this so any help would be very much appreciated.

Thank you.

-
-Reply
-

@@ -103,9 +85,6 @@ No clue how to print over the network, sorry. Thatâs actually why I set up a r

Would this work for brother hl-l2395dw scanner?

-Reply

@@ -117,9 +96,6 @@ No clue how to print over the network, sorry. Thatâs actually why I set up a r

Thanks so much for posting this. This helped get my HL-L2300D working properly.

-Reply

@@ -136,9 +112,6 @@ No clue how to print over the network, sorry. Thatâs actually why I set up a r

It was because I had my USB B plugged into the ethernet jack of my BR-2270DW.

Everything works now. Thank you so much.

-Reply

@@ -150,8 +123,5 @@ No clue how to print over the network, sorry. Thatâs actually why I set up a r

Any steps to do this with a wifi connected HL-2270DW? Thanks!

-Reply

- \ No newline at end of file + diff --git a/posts-comments/qr-backup.html b/comments/qr-backup.html similarity index 78% rename from posts-comments/qr-backup.html rename to comments/qr-backup.html index a8cbf80..29015c0 100644 --- a/posts-comments/qr-backup.html +++ b/comments/qr-backup.html @@ -10,9 +10,6 @@

nice! Iâve played with some similar ideas, using tar and QR Code output to a thermal printer. The used to be a thing (Twibright Optar, IIRC: itâs fallen off the web) that made full-page scannable codes that got an almost useful data density. But they werenât QR Codes, so needed their own decoder.

-Reply

@@ -24,9 +21,6 @@

Actually, I link to it in the FAQ, itâs still on the web. An even better version was âPaperbackâ, but itâs 9 years unmaintainedâIâm looking into seeing if there is a maintained Linux port. Both do a lot of things right, even if they have a slightly different goal (high data density, over ease-of-use and foolproof restore).

-
-Reply
-
@@ -39,10 +33,7 @@
Also, feel free to recommend me a good, cheap thermal printer. I tried to do a âpoloroidâ thing (take a picture of yourself with webcam, immediately print to thermal) and found that mine was shit and the heat overexposed unrelated parts. QR codes seem like a reasonable application, although Iâd be concerned about the longevity of thermal paper for backups (can easily fade in heat).

Edit: If I remember correctly, I wanted to make a thermal-paper typewriter for a zine?

-
-Reply
-

- \ No newline at end of file + diff --git a/posts-comments/running-a-forge-server-on-headless-linux.html b/comments/running-a-forge-server-on-headless-linux.html similarity index 71% rename from posts-comments/running-a-forge-server-on-headless-linux.html rename to comments/running-a-forge-server-on-headless-linux.html index 4d8bdbe..a4a4cb8 100644 --- a/posts-comments/running-a-forge-server-on-headless-linux.html +++ b/comments/running-a-forge-server-on-headless-linux.html @@ -10,9 +10,6 @@

An honest sharing about downloaing and installing Minecraft 1.6.4. I followed what you mentioned and did it sucessfully. Right now, Iâm playing with my little son and guiding him what the terrific things are. Anyway, thanks a lot!

-Reply

@@ -25,9 +22,6 @@

4am installs of servers is hard, and trying to follow the Forge wiki which is inaccurate at best is hard.
You got me from dead brick to running box in ten minutes. Thank you.

-Reply

@@ -39,9 +33,6 @@ You got me from dead brick to running box in ten minutes. Thank you.

Thanks so much! Iâve been looking for these steps for a while, and finally found something that works. Would have been first prize if it could be used along with a GUI frontend like mcmyadmin, but oh well ð

-Reply

@@ -53,9 +44,6 @@ You got me from dead brick to running box in ten minutes. Thank you.

Same here. Searched all over found yours and running in moments. My kids have been hooked on mods and wanted a server. I have VMware and can spin up a linux box up in moments. This was so easy. Thanks.

-Reply

@@ -67,9 +55,6 @@ You got me from dead brick to running box in ten minutes. Thank you.

Very useful information, Thank you

-Reply

@@ -81,9 +66,6 @@ You got me from dead brick to running box in ten minutes. Thank you.

Does anyone here have problems with installing the mods. Im donwloading them in the mods directory with the cmd âwgetâ and nothing works.Am i using the wrong command or what?

-Reply

@@ -95,9 +77,6 @@ You got me from dead brick to running box in ten minutes. Thank you.

Thanks for sharing Minecraft. Can you also share Roblox Apk?

-Reply

@@ -110,9 +89,6 @@ You got me from dead brick to running box in ten minutes. Thank you.

âone of the devs seems actively hostile around providing help to â¦â

this is so typical of the linux community. really. And I am a developer with 20 years experience, I hate the Linux community.

-Reply

@@ -124,8 +100,5 @@ You got me from dead brick to running box in ten minutes. Thank you.

lol this tutorial still works 9 years later thanks tho this helped so much!!

-Reply

- \ No newline at end of file + diff --git a/posts-comments/steak-tartare-3.html b/comments/steak-tartare-3.html similarity index 71% rename from posts-comments/steak-tartare-3.html rename to comments/steak-tartare-3.html index d9ef83f..189c21f 100644 --- a/posts-comments/steak-tartare-3.html +++ b/comments/steak-tartare-3.html @@ -10,8 +10,5 @@

So tasty! Enjoyed our month of obsessing over the recipe ^u^

-Reply

- - \ No newline at end of file + diff --git a/posts-comments/streaming-linux-twitch-using-ffmpeg-and-alsa.html b/comments/streaming-linux-twitch-using-ffmpeg-and-alsa.html similarity index 67% rename from posts-comments/streaming-linux-twitch-using-ffmpeg-and-alsa.html rename to comments/streaming-linux-twitch-using-ffmpeg-and-alsa.html index 3fa0d5a..8f478e0 100644 --- a/posts-comments/streaming-linux-twitch-using-ffmpeg-and-alsa.html +++ b/comments/streaming-linux-twitch-using-ffmpeg-and-alsa.html @@ -10,8 +10,5 @@

Thanks for your post.

-Reply

- - \ No newline at end of file + diff --git a/posts-comments/terminal-goal-rationality-techniques.html b/comments/terminal-goal-rationality-techniques.html similarity index 72% rename from posts-comments/terminal-goal-rationality-techniques.html rename to comments/terminal-goal-rationality-techniques.html index 78dd2d6..27a2980 100644 --- a/posts-comments/terminal-goal-rationality-techniques.html +++ b/comments/terminal-goal-rationality-techniques.html @@ -12,8 +12,5 @@ Murphy jitsu has potential.
The modified goal factor has a suspicious component of leaving things to future me that I donât like a bit though.

-Reply

- - \ No newline at end of file + diff --git a/posts-comments/time-management-optimizers-satisficers-minimizers.html b/comments/time-management-optimizers-satisficers-minimizers.html similarity index 75% rename from posts-comments/time-management-optimizers-satisficers-minimizers.html rename to comments/time-management-optimizers-satisficers-minimizers.html index 517e42b..89c1a0a 100644 --- a/posts-comments/time-management-optimizers-satisficers-minimizers.html +++ b/comments/time-management-optimizers-satisficers-minimizers.html @@ -10,8 +10,5 @@

Very nice! I often find myself automatically optimizing â the easiest place to see this is in Animal Crossing where Iâll keep a large cash of items for when villagers might ask for one. The idea is to always be prepared (I blame girl scouts). I suppose the negative to this behavior is that it could lead to hording if left unchecked.

-Reply

- - \ No newline at end of file + diff --git a/posts-comments/tiny-cute-vampire-bat.html b/comments/tiny-cute-vampire-bat.html similarity index 75% rename from posts-comments/tiny-cute-vampire-bat.html rename to comments/tiny-cute-vampire-bat.html index e0383cc..7dfa06c 100644 --- a/posts-comments/tiny-cute-vampire-bat.html +++ b/comments/tiny-cute-vampire-bat.html @@ -10,8 +10,5 @@

Hola! Just wanted to let you know I nominated you for the Versatile Blogger Award, for more info check out my post here: http://sometimesicook.net/2015/10/14/the-versatile-blogger-award/ ð

-Reply

- - \ No newline at end of file + diff --git a/posts-comments/understanding-gzip-2.html b/comments/understanding-gzip-2.html similarity index 80% rename from posts-comments/understanding-gzip-2.html rename to comments/understanding-gzip-2.html index a6ff23d..eb5a3ea 100644 --- a/posts-comments/understanding-gzip-2.html +++ b/comments/understanding-gzip-2.html @@ -12,9 +12,6 @@ 1:1100,2:0,4:1101,16:1110,17:1111,18:10â

Thereâs no explanation as to where you pulled these bits from, I understand the number of bits comes from the aforementioned 414442 pulled out prior to this but Iâve seen nothing that explains how you pulled out the bit values for each of these code words

-Reply

@@ -28,9 +25,6 @@
So first we assign 0 to the length-1 codeword (0 is lower than 1), then we assign the length-2 codeword 10 (the lowest length-2 code that doesnât have a prefix 0), and finally we assign the length-4 codewords from left to right (1100, 1101, 1110, 1111 donât have a prefix of 0 or 10 â they are the lowest and only codewords without one of those prefixes).

Thanks for the comment, it makes me happy to know anyone is reading this, and itâs very helpful to know whatâs unclear in my writeup. Iâd love to make it as good as possible.

-
-Reply
-
- @@ -55,9 +49,6 @@ }
  â`
  
  -
  -Reply
  -

Which doesnât follow that pattern, how do you determine the literals range?

-Reply

@@ -97,9 +85,6 @@ Byte 21-22: 111111 10 0001000 1. Copy â0â code length 11-138 times. 0b000100

Iâm pretty lost trying to follow this comment, sorry. To clarify, for the âdynamicâ compression, you first extract the CODING TABLE for the literals (the step youâre talking about). Then, you use the coding table to decompress the actual stream of content. Itâs a two-step process, which is why itâs so complex.

The key point I suspect youâre missed was: âLetâs read 267 numbers: the lengths of the codeword for each row.â These numbers are the length of the code for each literal in order (all of them, not just some of them). A length of 0 means ânot presentâ.

-Reply

@@ -111,9 +96,6 @@ Byte 21-22: 111111 10 0001000 1. Copy â0â code length 11-138 times. 0b000100

Was gonna say I already had the code table but when it printed it seems the codes got corrupted, Iâll have to get back to you after fixing it. Anyways I find âguidesâ that donât use real examples as this one does to be confusing sometimes, usually where it matters, thatâs why I was consulting you who provided those key examples, I got a heck of a lot further with your guide than any other Iâve tried following which didnât give the byte by byte, bit by bit example.

-Reply

@@ -210,9 +192,6 @@ PrintBytes( 0x556653fbf4a0, 0, 16 )
}
`

-Reply

@@ -229,8 +208,6 @@ printf( "num as a character = '%c'\n", num );

To see if num lined up, since it did Iâm guessing those ranges where all the wasted leaves/branches of the huffman tree.

-
-

@@ -253,9 +230,6 @@ printf( "num as a character = '%c'\n", num );

https://cboard.cprogramming.com/c-programming/180462-increment-gone-wrong.html#post1302477

The forum there is better able to handle pre-formatted code than this comment system so please take a look and tell me if you think Iâve mis-interpreted something also, and if so what you think is being mis-interpreted, in mean time Iâm going to try and clean up my code so I can upload to gitlab so that you can a clearer idea of where Iâm at and perhaps help me get to the last part, converting huffman codes to original values

-Reply

@@ -267,9 +241,6 @@ printf( "num as a character = '%c'\n", num );

Sorry, I donât really want to help someone else debug code or output. Iâd suggest following the links at the start of the blog article. Check out âinfgenâ in particular.

-Reply

@@ -281,9 +252,6 @@ printf( "num as a character = '%c'\n", num );

I originally gave up on infgen due to an access rights error when I tried installing via the package manager, the server refused to let me download it, then after your last comment I though to look for itâs github page, that one worked out fine, judging by itâs output the only thing Iâve got wrong are the length values for the distance symbols/codes/whatever you want to call them, where do you get that 2 from? same applies to the bit code along side them, where does that come from, is it just an iterated number for each valid symbol?

-Reply

@@ -297,9 +265,6 @@ printf( "num as a character = '%c'\n", num );

The first 260 codeword-lengths are used for one huffman table: 256 literals (always 256), then 1 âend of blockâ (always 2), then the remaining 3 are lengths.

The next 7 codeword-lengths are used to generate a second huffman table, for distances. The code generation works exactly the same as for the first table.

-
-Reply
-
- @@ -314,9 +279,6 @@ printf( "num as a character = '%c'\n", num );
  
  Because the previous method used that to determine what symbols to ignore until the code length increases, more specifically I had a âwhile ( cur_leng < max_leng )" loop and a sub loop that went looking for symbols expecting cur_leng codes and assigning them that way before using those same codes in another sub loop to decide if the next code should be increased further. Without knowing where those code lengths for the distance came from I can't progress.
  
  Thinking about it now I don't even have the codes for the length symbols, I just have the symbols. If it was supposed to be auto generated then I could understand the codes for the length symbols but I then don't understand the codes for the distance codes as they don't follow the huffman principle of no codes that can be misinterpreted.
  
  -
  -
  -
- @@ -328,8 +290,6 @@ printf( "num as a character = '%c'\n", num );
  
  Re-read above the table. The âbitsâ column is from bytes 24-25.
  
  -
  -
@@ -350,9 +310,6 @@ printf( "num as a character = '%c'\n", num );

No option to reply so Iâll do it here instead, I still donât see where the 2 came from, is that a minimum length or from another hard code table? So far I can only see the bits given as a set/unset symbol & code flag, thereâs no clear source for the length & extra bits values

-
-Reply
-
- @@ -369,9 +326,6 @@ printf( "num as a character = '%c'\n", num );
  â
  
  That was far too easily over looked, now I at least can generate the codes, though I think I will add a specialised handler and object to minimize code now that itâs getting more complicated than need be.
  
  -
  -Reply
  -

Iâve clearly misunderstood something somewhere, could you take a look at the code I outlined in the below post please and see if you can spot what Iâm misunderstanding, up until the point I have to lookup previously deflated values Iâve read the bits correctly but Iâve obviously not understood all the implied information correctly, I would post the code here but as youâve seen these comments donât support maintaining the code formatting resulting in a more confusing than necessary code.

https://cboard.cprogramming.com/c-programming/180462-increment-gone-wrong-post1302541.html#post1302541

-Reply

@@ -401,9 +352,6 @@ printf( "num as a character = '%c'\n", num );

Youâll be glad to know I finally got the algorithm right, have not looked at any source code from zlib or other projects so Iâm free to slap MIT License on it the whole way through once I convert it to a cleaner version of itself. The unclean version is here for anyoneâs reference.

https://gitlab.com/awsdert/uc-evidence/-/tree/9441a73e59834456c41c1049036fc60925b705a0

-
-Reply
-

âByte 10-11: 110 10011000 10010: A literal. 10011000 (152) minus 00110000 (48) is 104. 104 in ASCII is âhâ.â

Why are you subtracting 48?

-Reply

@@ -433,9 +378,6 @@ printf( "num as a character = '%c'\n", num );

The binary range (given above) is 00110000-10111111. Rather than decoding the binary value, we decode the offset within that range.

-
-Reply
-

1:111,2:1100,4:00,16:10,17:01,18:1101

Thanks!

-Reply

@@ -490,9 +429,6 @@ printf( "num as a character = '%c'\n", num );
}

-
-Reply
-

ââLiteralsâ 257-259 (all lengths) have codewords of length 4â Is this a hard and fast rule thatâs always true regardless of the data or is this true _just_ for the compressed string in this example?

-Reply

@@ -520,9 +453,6 @@ printf( "num as a character = '%c'\n", num );

Just for the example. If it was always true, we wouldnât have to encode it.

-
-Reply
-

âByte 27: 1110 10 0 1. Length 4. Whenever we read a length, we read a distance. The distance is a range, 7-8. The extra bit we read is 0b0=0, plus 7 is Distance 7. So we look back 7 bytes and copy 4. The new output is: baabbbabaabâ

I think there should be an extra a before the baabbbabaab

-Reply

@@ -551,9 +478,6 @@ printf( "num as a character = '%c'\n", num );

Iâve been reading quite a bit on DEFLATE in png files and I think this is the exact level of depth i needed to crack this whole mess. Thanks so much for writing this, sources like RFC1951 talk more about abstraction and general rules but to have a few examples lined out here its EXTREMELY useful. Thanks again

-
-Reply
-
- @@ -565,9 +489,6 @@ printf( "num as a character = '%c'\n", num );
  
  Small question about reading bits: for Huffman codes of the code itself defines a length range and then the extra bits are reversed so that they can be interpreted. But should I reverse the distance or the distance extra bits as well?
  
  -
  -Reply
  -

What if the distance was 2 and copying 4 symbols?

-Reply

@@ -601,10 +519,7 @@ Should I repeat the âbâ four times?
You just blindly copy characters starting DISTANCE back, but since youâre appending to the string, you never run out of symbols to copy.

If your string so far was âWow, copying is really neatâ, with distance 3 and copy 10 you would add: eateateate

-
-Reply
-

Out of curiosity, why did you stop the service?

-Reply

@@ -25,10 +22,7 @@
This was launched in 2015, and youâre the first person to ask me about the the service since. That should give you an idea of the popularity. Iâm very bad at marketing, to be fair, so itâs not totally clear that means the service was undesirable.

So when I eventually ran into some problem with active maintenance (the site needed updating to keep working, or something of that kind), I just didnât.

-
-Reply
-

The best practice or goal emphasized above with respect to templates and views is KISS and DRY. As long as the implementation does not become overly complex and difficult to grok, keep the template code DRY, otherwise KISS principle overrides the need to have template code that does not repeat itself.
-
âVertebrae Framework

A nice illustration of conflicting positive principles and resolution.

Very, very early in my self-improvement journey, I tried adding a 20 minute timer to my life. Every 20 minutes of my waking life, it went off. Iâd explicitly say to myself what it was I was doing (âWatching TVâ or âTalking to Fredâ). Itâs important to have a verbal or written note of what youâre doing.

I added a specific prompt at one point: âWhat am I doing, and why?â. I donât think goals are how I think about things today, but itâs how I tried to think back then (goal factoring). Today I might ask, âWhat am I doing, and do I want to be doing it? Do I want to be doing it this way?â

The basic 20-minute check-in process was very helpful to me. Some benefits:

Ding! What am I doing? It gave me an awareness of time. How long do things last? Our subjective sense of time doesnât always match. For example, doing my daily chores takes about 20 minutes. A typical conversation with a friend takes 60-120 minutes for me. But the chores feel way longer to me!
What am I doing, and do I want to be doing it? Interrupting default activities (a type of inertia). Watching TV until I get bored of it takes me 2-10 hours, if I donât have a timer. Having a timer interrupt me let me say âwell, maybe this will be the last episode, then.â Itâs easy to get into a low-energy state for leisure activities where you donât notice the passage of time. And itâs important to note, I donât have any rules. I can keep watching TV all night if I want to. This helps me avoid doing it out of pure inertia. (It also really keyed me in that TV does not really ârechargeâ my batteries. Other relaxation is better.)
Do I want to be doing it, or doing it this way? Interrupting bad approaches (a type of inertia). Sometimes, I get stubborn. Iâll keep trying to solve a problem in one way, for way too long. Having a regular interruption keys me into how long Iâve been doing that. Iâll notice I should maybe try a different approach or a work-around. Or give up.
Ding! Providing a check-in. Iâll just take stock, and say âhey, am I really having fun? is this even useful?â. Sometimes Iâm just doing something dumb. Or sometimes I forget to have fun or take a break for too long. This is my reminder to check in with my strategic system; my emotions; my body.

Hereâs a list of books I read in 2020. The ones in bold I recommend.

Fiction:

A College of Magics by Caroline Stevermer
A Crucible of Souls by Mitchell Hogan
Alcatraz and the Evil Librarians by Brandon Sanderson
A Memory Called Empire, by Arkady Martine
Apex (Nexus 3) by Ramez Naam
A Practical Guide to Evil, to end of book 5
Arena by Holly Jennings
Ariel by Steven Barnett
Ascend Online by Luke Chmilenko
Bastard Operator from Hell
Circe, by Madeline Miller
City of Brass by S A Chakrabarty p1-460
Cold Comfort Farm by Stella Gibbons
Colour out of Space by HP Lovecraft
Crux (Nexus 2) by Ramez Naam
Cryptonomicon by Neal Stephenson
Cultivation Chat Group â ch1-56
Dark Lord of Derholm by Dianna Wynne Jones
Dayworld by Philip Jose Farmer
Dayworld Rebel by Philip Jose Farmer # gave up halfway
Dust by Hugh Howey
Emperor Mage by Tamora Pierce
Enchantress by James Maxwell
Exhalation by Ted Chiang
Fall by Neal Stephenson p1-545
Forging Divinity by Andrew Rowe
Future Indefinite by Dave Duncan
Futuristic Tales of the Here and Now by Cory Doctrow
Ghostwater by Will Wight
Gideon the Ninth by Tansyn Muir
House of Blades by Will Wight
House of Earth and Blood by Sarah Maas
Ithenalinâs Restoration by Lawrence Watt-Evans
Lament by Maggie Stiefvater
Legacy of the Fallen by Luke Chmilenko p1-316
Lone Wolf / Kai adventure series 1-5, magnakai 1, by Joe Dever
Magic for Liars by Sarah Gailey
Magician by Raymond Feist
Magicians by Lev Grossman
Making Money by Terry Pratchett
Mirror Gate by Jeff Wheeler
New York Fantastic by Paula Guran
Nexus by Ramez Naam
Night of Madness by Lawrence Watt-Evans
Ninth House by Leigh Bardogo
Od Magic by Patricia McKillip p1-222
One Word Kill by Mark Lawrence
On the Shoulders of Titans by Andrew Rowe
Past Imperative by Dave Duncan
Piranesi by Susanna Clarke
Present Tense by Dave Duncan
Prince of Thorns by Mark Lawrence
Priory of the Orange Tree by Samantha Shannon, p1-534?
Rage of Dragons by Evan Winter (some)
Relics of War by Lawrence Watt-Evans
Starfish (Rifters 1) by Peter Watts
Shades of Milk and Honey by Mary Robinette Kowal (all)
Shift (Silo 6-8) by Hugh Howey
Shining Path by Matthew Skala
Shouldnât You Be In School? by Lemony Snicket
Sister Sable, by T Mountebank, p1-378
Skysworn by Will Wight
Skyward by Brandon Sanderson
Snowspelled by Stephanie Burges
Spellmonger by Terry Mancour, p1-165
Starfish by Peter Watts
Stone Unturned by Lawrence Watt-Evans
Storm Glass by Jeff Wheeler
Sufficiently Advanced Magic by Andrew Rowe
The Alienâs Lover by Zoey Draven
The Archived by Victoria Schwab
The Atrocity Archive by Charles Stross
The Blood of a Dragon by Lawrence Watt-Evans
The Burning White (Lightbringer 5) by Brent Weeks
The Collapsing Empire by John Scalzi
The Diamond Age by Neal Stephenson
The Fractured World by David Aries
The Goblin Emperor by Katherine Addison
The Library at Mount Char by Scott Hawkins
The Magic Goes Away by Larry Niven
The Maker of Universes by Philip Jose Farmer
The Misenchanted Sword by Lawrence Watt-Evans
The Mysterious Study of Doctor Sex by Tamsyn Muir
The Necromancerâs House by Christopher Buehlman
The Queenâs Poisoner by Jeff Wheeler
The Rook by Daniel OâMalley
The Sorcererâs Widow by Lawrence Watt-Evans
The Spell of the Black Dagger by Lawrence Watt-Evans
The Spriggan Mirror by Lawrence Watt-Evans
The Unwilling Warlord by Lawrence Watt-Evans
The Vondish Ambassador by Lawrence Watt-Evans
The Warrior Heir by Cinda Williams Chima, p1-116
The Wiz Biz by Rick Cook
The Woven Ring by MD Presley, p1-28
Three-Body Problem by Cixin Liu
Three Men in a Boat by Jerome K. Jerome
Twig by wildbow (arc 1-18)
Uncrowned by Will Wight
Underlord by Will Wight
Unsong by Scott Alexander
Unsouled by Will Wight
When Did You See Her Last? by Lemony Snicket
Wintersteel by Will Wight
With a Single Spell by Lawrence Watt-Evans
Wool by Hugh Howey (v1-5)

Nonfiction (mostly I read web nonfiction):

507 Mechanical Movements by Henry T Brown
Advanced Magick for Beginners by Alan Chapman
Broadcast Channels with Confidential Messages
Busy Beaver Frontier by Scott Aaronson. I did some work based on it.
Computational Geometry by Mark de Berg
Craeft by Alexander Langlands
D&D 5e Playerâs Handbook
D&D 5e Dungeon Masterâs Guide
Forrest Memâs Notebook
Forrest Mimâs Engineerâs Notebook
Forrest Mimâs Mini Notebook
Intelâs x86-64 manual
Introduction to Analysis by Maxwell Rosenlicht
Kademelia by Peter Maymounkov
kleiman v wright australian tax document
Incremental String Searching by Bertrand Meyer (KNP algorithm)
Rules to One Night Ultimate Werewolf
The Art of Computer Programming, v1, v3 by Donald Knuth (parts)
The Pragmatic Programmer
The Rust Programming Language
Thereâs Plenty of Room at the Bottom by Richard Feynman
Total Money Makeover by Dave Ramsey
W65025 manual (6502 clone)

What happened in 2020? Well,

(General news) COVID-19 of course, and Trump left office
I stayed inside. Iâve been getting groceries delivered, evenâIâve been somewhere other than my house maybe twice since COVID-19 lockdown started.
I started watching wayyy more videos, especially video game streams.
I looked into buying land in Colorado and living in an RV
I transcribed my log books, and started coverting them all to a standard, computer-parsable format (mostly done, one left).
I deleted bs.
I figured out twitch streaming, both with a standalone capture card and on linux.
I got hardware random number generators to work.
I designed v1 and v2 of a protocol to allow a set of computers to store a large amount of content. Itâs designed to back up things like the Internet Archive. Iâm calling the project âvalhallaâ, after ArchiveTeamsâs project valhalla and IA.BAK.
I learned to use an oscilloscope, and bit-banged SPI and I2C for a while, trying to get a 9-axis sensor to work unsuccessfully.
I learned how to make a pretty good pizza
I played a bunch of video games
I worked on the Lazy Beaver problem, and tied the state of the art.
I made a master TODO list, and finished every single TODO I had that took an hour or less.
I figured out how to make VMs in Linux and run them all the time
I got a tablet, and learned GIMP and InkScape well enough to draw some stuff.
I wrote a custom client for omegle
I did a yearly backup
I did various research. I learned about algorithms, data structures, RALA, and quantum physics.
I wrote up my cookbook and released it.
I wrote some blog posts ð
Four of my friends moved to Ohio, two from nearby me. I only know one person in the state Iâm in well at this point.
A friend of mine got out of jail and got to go home.

In 2020 Iâm newly retired, so Iâve had free time. I think itâs fun to do reviews, so without further ado hereâs every video game I played in 2020!

I recommend:

(4/5) Among Us â Very fun. Itâs only fun with voice chat with friends, so Iâve only gotten to play once or twice. Iâve been watching it more than playing it. Also free to play for mobile gamersâIâm tired of the âeveryone buys a copyâ model of group gameplay.
(4/5) Brogue. Brogue is an ascii-art roguelike. Itâs great, and it has a nice difficulty ramp. Itâs a good âquick breakâ game. I play it in preference to other roguelikes partly because I havenât done it to death yet, and partly because I donât need a numpad?
(4/5) Cook Serve Delicious 3. One of the more fun games I played this year. You get really into it, but I had trouble relaxing and paying attention to the real world when I played too much, haha. I own but havenât played the first twoâI gather this is pretty much just a refinement.
(4/5) Green Hell. Price tag is a bit high for the number of hours I got out of it, but I havenât finished the story. Great graphics, and the BEST map design Iâve seen in a 3D game in a long time. It feels like a real place, with reasonable geography instead of copy-pasted tiles. I love that as you walk along, you can just spot a cultivated area from the rest of the jungleâit feels more like itâs treating me like an adult than most survival games. Everything still gets highlighted if you can pick it up. I played the survival mode, which was okay but gets old quickly. I started the story modeâI think it would be fine, but it has some LONG unskippable scenes at the start, including a very hand-holdy tutorial, that I think they should have cut. I did start getting into the story and was having fun, but I stopped. I might finish the game some time.
(4/5) Hyperrogue. One of my recent favorites. The dev has made a fair number of highly experimental games, most of which are a total miss with me, but this one is fun. I do wish the early game wasnât quite as repetitive. Failing another solution, I might actually want this not to be permadeath, or to have a save feature? I bought it on steam to support the dev and get achievements, but itâs also available a version or two behind free, which is how I tried it. Constantly getting updates and new worlds.
(4/5) Minecraft â Compact Claustrophobia modpack. Fun idea, nice variety. After one expansion felt a little samey, and it was hard to start with two people. Iâd consider finishing this pack.
(4/5) Overcooked 2. Overcooked 2 is just more levels for Overcooked. The foods in the second game is more fun, and it has better controls and less bugs. If youâre considering playing Overcooked, I recommend just starting with the second game, despite very fun levels in the first. I especially appreciate that the second game didnât just re-use foods from the first.
(4/5) Please Donât Press Anything. A unique little game where you try to get all the endings. I had a lot of fun with this one, but it could have used some kind of built-in hints like Reventure. Also, it had a lot of red herrings. Got it for $2, which it was well worth.
(5/5) Reventure. Probably the best game new to me this year. Itâs a short game where you try to get each of about 100 endings. The art and writing are cute and funny. The level design is INCREDIBLE. One thing I found interesting is the early prototypeâif I had played it, I would NOT have imagined it would someday be any fun at all, let alone as amazing as it is. As a game designer I found that interesting! I did 100% complete this oneâthereâs a nice in-game hint system, but there were still 1-3 âhuhâ puzzles, especially in the post-game content, one of which I had to look up. Itâs still getting updates so Iâm hoping those will be swapped for something else.
(5/5) Rimworld. Dwarf fortress, but with good cute graphics, set in the Firefly universe. Only has 1-10 pawns instead of hundreds of dwarves. Basically Dwarf Fortress but with a good UI. I wish you could do a little more in Rimworld, but itâs a fantastic, relaxing game.
(5/5) Slay the Spire. Probably the game I played most this year. A deckbuilding adventure through a series of RPG fights. A bit luck-based, but relaxing and fun. I like that you can play fast or slow. Very, very well-designed UIâyou can really learn how things work. My favorite part is that because itâs singleplayer, itâs really designed to let you build a game-breaking deck. Thatâs how it should be!
(4/5) Stationeers. I had a lot of fun with this one. Itâs similar to Space Engineers butâ¦ fun. It has better UI by a mile too, even if itâs not perfect. I lost steam after playing with friends and then going back to being alone, as I often do for base-building games. Looks like you can genuinely make some complicated stuff using simple parts. Mining might not be ideal.
(5/5) Spy Party. One of my favorite games. Very fun, and an incredibly high skill ceiling. Thereâs finally starting to be enough people to play a game with straners sometimes. Bad support for âhot seatââI want to play with beginners in person, and it got even harder with the introduction of an ELO equivalent and removing the manual switch to use âbeginnerâ gameplay.
(4/5) Telling Lies. A storytelling game. The core mechanic is that you can use a search engine for any phrase, and it will show the top 5 survellance footage results for that. The game internally has transcripts of every video. I didnât really finish the game, but I had a lot of fun with it. The game was well-made. I felt the video acting didnât really add a huge amount, and they could have done a text version, but I understand it wouldnât have had any popular appeal. The acting was decent. Thereâs some uncomfortable content, on purpose.
(4/5) Totally Accurate Battle Simulator (TABS). Delightful. Very silly, not what youâd expect from the name. What everyone should have been doing with physics engines since they were invented. Imagine that when a caveman attacks, the club moves on its own and the caveman just gets ragdolled along, glued to it. Also the caveman and club have googley eyes. Donât try to win or it will stop being fun. Learn how to turn on slo-mo and move the camera.
(4/5) We Were Here Together. Lots of fun. I believe the second game out of three. Still some crashes and UI issues. MUCH better puzzles and the grpahics are gorgeous. They need to fix the crashes or improve the autosave, we ended up replaying a lot of both games from crashes. Itâs possible I should be recommending the third game but I havenât played it yet.

The Rest

(3/5) 5D Chess with Multiverse Time Travel. More fun that it sounds. If you play to mess around and win by accident, itâs pretty good. Definitely play with a second human player, though.
(1.5/5) 7 billion humans. Better than the original, still not fun. Soulless game about a soulless, beige corporation. Just play Zachtronics instead. If youâre on a phone and want to engage your brain, play Euclidea.
(3/5) A Dark Room. Idle game.
(1/5) Amazing Cultivation Simulator. A big disappointment. Bad english voice acting which canât be turned off, and a long, unskippable tutorial. I didnât get to actual gameplay. I like Rimworld and cultivation novels so I had high hopes.
(3/5) ADOM (Steam version) â Fun like the original, which I would give 5/5. Developed some major issues on Linux, but I appreciate that thereâs a graphical version available, one of my friends will play it now.
(4/5) agar.io â Good, but used to be better. Too difficult to get into games now. Very fun and addictive gameplay.
(3/5) Amorous â Furry dating sim. All of the hot characters are background art you canât interact with, and the characters you can actually talk to are a bunch of sulky nerds who for some reason came to a nightclub. I think it was free, though.
(0/5) Apis. Alpha game, AFAIK I was the first player. Pretty much no fun right now (to the point of not really being a game yet), but it could potentially become fun if the author puts in work.
(4/5) Autonauts. I played a ton of Autonauts this year, almost finished it, which is rare for me. My main complaint is that itâs fundamentally supposed to be a game about programming robots, but I canât actually make them do more than about 3 things, even as a professional programmer. Add more programming! It can be optional, thatâs fine. Theyâre adding some kind of tower defense waves instead, which is bullshit. Not recommended because itâs not for everyone.
(3/5) A-Z Inc. Points for having the guts to have a simple game. At first this looked like just the bones of Swarm Simulator, but the more you look at the UI and the ascension system, the worse it actually is. I would regularly reset because I found out an ascension âperkâ actually made me worse off.
(5/5) Beat Saber. Great game, and my favorite way to stay in shape early this year. Oculus VR only, if you have VR you already have this game so no need to recommend. Not QUITE worth getting a VR set just to play it at current prices.
(1/5) Big Tall Small. Good idea, but no fun to play. Needed better controls and level design, maybe some art.
(0.5/5) Blush Blush. Boring.
(3/5) Business Shark. I had too much fun with this simple game. All you do is just eat a bunch of office workers.
(3/5) chess.com. Turns out I like chess while Iâm high?
(3/5) Circle Empires Rivals. Decent, more fun than the singleplayer original. It shouldnât really have been a separate game from Circle Empires, and Iâm annoyed I couldnât get it DRM-free like the original.
(3/5) Cross Virus. By Dan-box. Really interesting puzzle mechanics.
(4/5) Cultist Simulator. Really fun to learn how to playâI love games that drop you in with no explanation. Great art and writing, I wish I could have gotten their tarot deck. Probably the best gameplay âambienceâ Iâve seenâgetting a card thatâs labeled âfleeting sense of radianceâ that disappears in 5 seconds? Great. Also the core stats are very well thought out for âfeelâ and real-life accuracyâdread (depression) conquers fascination (mania), etc. It has a few gameplay gotchas, but theyâre not too bigâlayout issues, inability to go back to skipped text, or to put your game in an unwinnable state early on). Unfortunately itâs a âroguelikeâ, and itâs much too slow-paced and doesnât have enough replay value, so it becomes a horrible, un-fun grind when you want to actually win. I probably missed the 100% ending but I wonât be going back to get it. I have no idea who would want to play this repeatedly. Iâm looking forward to the next game from the same studio though! I recommend playing a friendâs copy instead of buying.
(2/5) Darkest Dungeon. It was fine but I donât really remember it.
(2/5) Dicey Dungeons. Okay deck-building roguelike gameplay (with an inventory instead of a deck). Really frustrating, unskippably slow difficulty curve at the start. I played it some more this year and liked it better because I had a savegame. I appreciate having several character classes, but they should unlock every difficulty from the start.
(2/5) Diner Bros. Basically just a worse Overcooked. I didnât like the controls, and it felt too repetitive with only one diner.
(2/5) Donât Eat My Mind You Stupid Monster. Okay art and idea, the gameplay wasnât too fun for me.
(2/5) Donât Starve â Iâve played Donât Stave maybe 8 different times, and itâs never really gripped me, I always put it back down. Itâs slow, a bit grindy, and thereâs no bigger goalâall you can do is live.
(3/5) Donât Starve Together â Confusingly, Donât Starve Together can be played alone. Itâs Donât Starve, plus a couple of the expansions. This really could be much more clearly explained.
(1/5) Elemental Abyss â A deck-builder, but this time itâs grid-based tactics. Really not all that fun. Just play Into the Abyss instead or something.
(1/5) Else Heart.Break() â I was excited that this might be a version of âHack Nâ Slashâ from doublefine that actually delivered and let you goof around with the world. I gave it up in the first ten minutes, because the writing and characters drove me crazy, without getting to hacking the world.
(2/5) Everything is Garbage. Pretty good for a game jam game. Not a bad use of 10 minutes. I do think itâs probably possible to make the game unwinnable, and the ending is just nothing.
(1/5) Evolve. Idle game, not all that fun. I take issue with the mechanic in Sharks, Kittens, and this where buying your 15th fence takes 10^15 wood for some reason.
(4/5) Exapunks. Zachtronics has really been killing it lately, with Exapunks and Opus Magnum. WONDERFUL art and characters during story portions, and much better writing. The gameplay is a little more varied than in TIS-100 or the little I played of ShenZen I/O. My main complaint about Zachtronics games continues to be, that I donât want to be given a series of resource-limited puzzles (do X, but without using more than 10 programming instructions). Exapunks is the first game where it becomes harder to do something /at all/, rather than with a particular amount of resources, but itâs still not there for me. Like ShenZen, they really go for a variety of hardware, too. Canât recommend this because itâs really only for programmers.
(1/5) Exception. Programming game written by some money machine mobile games company. Awful.
(4/5) Factorio. Factorioâs great, but for me it doesnât have that much replay value, even with mods. I do like their recent updates, which included adding blueprints from the start of the game, improving belt sorting, and adding a research queue. We changed movement speed, made things visually always day, and adding a small number of personal construction robots from the start this run. Iâm sure if youâd like factorio youâve played it already.
(3/5) Fall Guys â I got this because it was decently fun to watch. Unfortunately, itâs slightly less fun to play. Overall, thereâs WAY too much matchmaking waiting considering the number of players, and the skill ceiling is very low on most of the games, some of which are essentially luck (Iâm looking at you, team games).
(3/5) Forager â Decent game. A little too much guesswork in picking upgradesâwas probably a bit more fun on my second play because of that. Overall, nice graphics and a cute map, but the gameplay could use a bit of work.
(3/5) Getting Over It â Funny idea, executed well. Pretty sure my friends and I have only gotten through 10% of the game, and all hit about the same wall (the first tunnel)
(3/5) Guild of Dungeoneering â Pretty decent gameplay. I feel like itâs a bit too hard for me, but thatâs fine. Overall I think it could use a little more cute/fun art, I never quite felt that motivated.
(1/5) Hardspace: Shipbreakers. Okay, I seriously didnât get to play this one, but I had GAMEBREAKING issues with my controller, which is a microsoft X-box controller for PCâTHE development controller.
(2/5) Helltaker. All right art, meh gameplay. But eh, itâs free!
(3/5) Hot Lava. Decent gameplay. Somehow felt like the place that made this had sucked the souls out of all the devs firstâno one cared about the story or characters. Itâs a game where the floor is made out of lava, with a saturday morning cartoon open, so that was a really an issue. Admirable lack of bugs, though. Iâm a completionist so I played the first world a lot to get all the medals, and didnât try the later ones.
(3/5) House Flipper â Weird, but I had fun. I wish the gameplay was a little more unifiedâit felt like a bunch of glued-together minigames.
(2/5) Hydroneer. Utterly uninspiring. I couldnât care about making progress at all, looked like a terrible grind to no benefit.
(1/5) io. Tiny game, I got it on Steam, also available on phone. Basically a free web flash game, but for money. Not good enough to pay the $1 I paid. Just a bit of a time-killer.
(3/5) Islanders â All you do is place buildings and get points. Not particularly challenging, but relaxing. Overall I liked it.
(3/5) Jackbox â I played this online with a streamer. Jackbox has always felt a little bit soulless money grab to me, but itâs still all right. I like that I can play without having a copyâwe need more games using this purchase model.
(3/5) Life is Feudal â Soul-crushingly depressing and grindy, which I knew going in. I thought it wasâ¦ okay, but I really want an offline play mode (Yes, I know thereâs an unsupported single-player game, but itâs buggier and costs money). UI was pretty buggy, and I think hunting might literally be impossible.
(2/5) Minecraft â Antimatter Chemistry. Not particularly fun.
(3/5) Minecraft â ComputerCraft. I played a pack with just ComputerCraft and really nothing else. Was a little slow, would have been more fun with more of an audience. I love the ComputerCraft mod, I just didnât have a great experience playing my pack I made.
(3/5) Minecraft â Foolcraft 3. Fun, a bit buggy. Honestly I canât remember it too well.
(1/5) Minecraft â Manufactio. Looked potentially fun, but huge bugs and performance issues, couldnât play.
(4/5) Minecraft â Tekkit. Tekkit remains one of my favorite Minecraft modpacks.
(3/5) Minecraft â Valhelsia 2. I remember this being fun, but I canât remember details as much as Iâd like. I think it was mostly based around being the latest version of minecraft?
(4/5) Minecraft â Volcano Block. Interesting, designed around some weird mods I hadnât used. I could have used more storage management or bulk dirt/blocks early in the gameâfelt quite cramped. Probably got a third of the way through the pack. I got novelty value out of it, but I wouldnât have enjoyed it if I had ever used the plant mod beforeâitâs a very fixed, linear progression.
(5/5) Minit. This is a weird, small game. I actually had a lot of fun with it. Then I 100% completed it, which was less fun but I still had a good time overall.
(3/5) Monster Box. By Dan-box. One of two Dan-box games I played a lot of. Just visually appealing, the gameplay isnât amazing. Also, Dan-box does some great programmingâthis is a game written in 1990 or so, and it can render hundreds of arrows in the air smoothly in a background tab.
(3/5) Monster Train. A relatively fun deckbuilding card game. It canât run well on my computer, which is UNACCEPTABLEâthis is a card game with 2D graphics. My MICROWAVE should run this shit in 2020. Ignoring that, the gameplay style (summon monsters, MTG style) just isnât my cup of tea.
(2/5) Moonlighter. Felt like it was missing some inspiration, just didnât have a sense of âfunâ. The art was nice. The credits list is surprisingly long.
(2/5) Muse Dash. All right, a basic rhythm game. Not enough variety to the game play, and everything was based around perfect or near-perfect gameplay, which makes things less fun for me.
(3/5) NES games â various. Dr Mario, Ice Climbers. Basically, I got some Chinese handheld âgameboyâ that has all the NES games preloaded on it. Overall it was a great purchase.
(2/5) Noita. âThe Powder Gameâ by Dan-Box, as a procedurally generated platformer with guns. Lets you design your own battle spells. Despite the description, you really still canât screw around as much as Iâd like. I also had major performance issues
(3/5) Observation. I havenât played this one as much as Iâd like, I feel like it may get better. Storytelling, 3D game from the point of view of the AI computer on a space station. I think I might have read a book itâs based on, unfortunately.
(2/5) One Step From Eden. This is a deck-building combat tactics game. I thought it was turn-based, but itâs actually realtime. I think if it was turn-based I would have liked it. The characters were a bit uninspired.
(1/5) Orbt XL. Very dull. I paid $0.50 for it, it was worth that.
(4/5) Opus Magnum. Another great game from Zachtronics, along with Exapunks theyâre really ramping up. This is the third execution of the same basic concept. Iâd like to see Zachtronics treading new ground more as far as gameplayâthat said, it is much improved compared to the first two iterations. The art, writing, and story were stellar on the other hand.
(3/5) Out of Space. Fun idea, you clean a spaceship. Itâs never that challenging, and it has mechanics such that it gets easier the more you clean, rather than harder. Good but not enough replay value. Fun with friends the first few times. The controls are a little wonky.
(1/5) Outpost (tower defense game). I hate all tower defense.
(3/5) Overcooked. Overcooked is a ton of fun.
(4/5) Powder Game â Dan-box. I played this in reaction to not liking Noita. Itâs fairly old at this point. Just a fun little toy.
(1/5) Prime Mover â Very cool art, the gameplay put me to sleep immediately. A âcircuit builderâ game but somehow missing any challenge or consistency.
(2/5) Quest for Glory I. Older, from 1989. Didnât really play this much, I couldnât get into the writing, and the pseudo-photography art was a little jarring.
(4/5) Raft. I played this in beta for free on itch.io, and had a lot of fun. Not enough changed that it was really worth a replay, but it has improved, and I got to play with a second player. Not a hard game, which I think was a good thing. The late game theyâve expanded, but it doesnât really add much. The original was fun and so was this.
(3/5) Satisfactory. I honestly donât know how I like this oneâI didnât get too far into it.
(4/5) Scrap Mechanic. I got this on a recommendation from a player who played in creative. I only tried the survival modeâthat mode is not well designed, and their focuses for survival are totally wrong. I like the core game, you can actually build stuff. If I play again, Iâll try the creative mode, I think.
(3.5/5) Shapez.io. A weird, abstracted simplification of Factorio. If I hadnât played factorio and half a dozen copies, I imagine this would have been fun, but itâs just more of the same. Too much waitingâblueprints are too far into the game, too.
(2.5/5) Simmiland. Okay, but short. Used cards for no reason. For a paid game, I wanted more gameplay out of it?
(0.5/5) Snakeybus. The most disappointing game I remember this year. Someone made âSnakeâ in 3D. There are a million game modes and worlds to play in. I didnât find anything I tried much fun.
(1/5) Soda Dungeon. A âmobileâ (read: not fun) style idle game. Patterned after money-grab games, although I donât remember if paid progress was actually an option. I think so.
(4/5) Spelunky. The only procedurally generated platformer Iâve ever seen work. Genuinely very fun.
(4/5) Spelunky 2. Fun, more of an upgrade of new content than a new game. Better multiplayer. My computer canât run later levels at full speed.
(1/5) Stick Ranger 2. Dan-box. Not much fun.
(3/5) Superliminal. Fun game. A bit short for the pricetag.
(3/5) Tabletop Simulator â Aetherâs End: Legacy. Interesting, a âcampaignâ (series of challenge bosses and pre-written encounters) deckbuilding RPG. I like the whole âcampaign RPG boardgameâ idea. This would have worked better with paper, there were some rough edges in both the game instructions and the port to Tabletop Simulator.
(4/5) Tabletop Simulator â The Captain is Dead. Very fun. Iâd love to play with more than 2 people. Tabletop simulator was so-so for this one.
(2/5) Tabletop Simulator â Tiny Epic Mechs. You give your mech a list of instructions, and it does them in order. Arena fight. Fun, but I think I could whip up something at least as good.
(3/5) The Council. One of the only 3D games I finished. Itâs a story game, where you investigate whatâs going on and make various choices. Itâs set in revolutionary france, at the Secret World Council that determines the fate of the world. It had a weak ending, with less choice elements than the rest of the game so far, which was a weird decision. Also, it has an EXCRUTIATINGLY bad opening scene, which was also weird. The middle 95% of the game I enjoyed, although the ending went on a little long. The level of background knowledge expected of the player swung wildlyâthey seemed to expect me to know who revolutionary French generals were with no explanation, but not Daedalus and the Minotaur. The acting was generally enjoyableâthereâs a lot of lying going on in the game and itâs conveyed well. The pricetag is too high to recommend.
(0/5) The Grandmaâs Recipe (Unus Annus). This game is unplayably badâitâs just a random pixel hunt. Maybe it would be fun if you had watched the video itâs based on.
(3/5) The Room. Pretty fun! I think this is really designed for a touchscreen, but I managed to play it on my PC. Played it stoned, which I think helps with popular puzzle gamesâit has nice visuals but itâs a little too easy.
(3/5) This Call May Be Recorded. Goofy experimental game.
(4/5) TIS-100. Zachtronics. A programming game. I finally got done with the first set of puzzles and into the second this year. I had fun, definitely not for everyone.
(3/5) Trine. I played this 2-player. I think the difficulty was much better 2-player, but it doesnât manage 2 players getting separated well. Sadly we skipped the story, which seemed like simple nice low-fantasy. Could have used goofier puzzles, it took itself a little too seriously and the levels were a bit same-y.
(2/5) Unrailed. Co-op railroad building game. It was okay but there wasnât base-building. Overall not my thing. Iâd say I would prefer something like Overcooked if itâs going to be timed? Graphics reminded me of autonauts.
(2/5) Vampire Night Shift. Art game. Gameplay could have used a bit of polish. Short but interesting.
(4/5) Wayward. To date, the best survival crafting system Iâve seen. You can use any pointy object and stick-like object, together with glue or twine, to make an arrow. The UI is not great, and thereâs a very counter-intuitive difficulty system. You need to do a little too much tutorial reading, and it could use more goals. Overall very fun. Under constant development, so how it plays a given week is a crapshoot. The steam version finally works for me (last time I played it was worse than the free online alpha, now itâs the same or better). I recomend playing the free online version unless you want to support the author.
(1/5) We Need to Go Deeper. Multiplayer exploration game in a sub, with sidescrolling battle. Somehow incredibly unfun, together with high pricetag. Aesthetics reminded me of Donât Starve somehow.
(2/5) We Were Here. Okay 2-player puzzle game. Crashed frequently, and there were some âhuhâ puzzles and UI. Free.
(3/5) Yes, your grace. Gorgeous pixel art graphics. The story is supposed to be very player-dependent, but I started getting the feeling that it wasnât. I didnât quite finish the game but I think I was well past halfway. Hard to resume after a save, you forget things. I got the feeling I wouldnât replay it, which is a shame because itâs fun to see how things go differently in a second play with something like this.

These are not all new to me, and very few came out in 2020. I removed any games I donât remember and couldnât google (a fair number, I play a lot of game jam games) as well as any with pornographic content.

Hereâs a list of books I read in 2021. The ones in bold I recommend.

Fiction:

Enigma by Graeme Base
City of Stairs by Robert Jackson Bennett
Look to Windward (Culture 7) by Ian Banks
Surface Detail (Culture 8) by Ian M Banks
Pump Six by Paolo Bacigalupi
Six of Crows by Leigh Bardugo
Lexicon by Max Barry
Mage Errant 1 by John Bierce
Mage Errant 2 by John Bierce
Mage Errant 3 by John Bierce
Mage Errant 4 by John Bierce
Mage Errant 5 by John Bierce
The Atlas Six by Olivie Blake
Lilithâs Brood (Xenogenesis 1) by Octavia E Butler
Elegy Beach (Change 2) by Steven Boyett
Curse of Charion by Louis Bujold
Xenocide by Orson Scott Card
Bohemian Gospel by Dan Carpenter
Convergence (Foreigner 18) by C J Cherryh
Emergence (Foreigner 19) by C J Cherryh
Convergence (Foreigner 21) by C J Cherryh
Iron Prince by Bryce OâConner and Luke Chmilenko
Murder on the Orient Express by Agatha Christie
The Alchemist by Paulo Coelho
Artemis Fowl (Artemis Fowl 1) by Eoin Colfer
The Arctic Incident (Artemis Fowl 2) by Eoin Colfer
Eternity Code (Artemis Fowl 3) by Eoin Colfer
Opal Deception (Artemis Fowl 4) by Eoin Colfer
Space Between Worlds by J Conrad and Micaiah Johnson
Little Brother by Cory Doctrow
Homeland (Little Brother 2) by Cory Doctrow
Children of Chaos by Dave Duncan
The Alchemistâs Apprentice by Dave Duncan
The Alchemistâs Code by Dave Duncan
The Alchemistâs Pursuit by Dave Duncan
The Cutting Edge by Dave Duncan
Upland Outlaws by Dave Duncan
The Stricken Field by Dave Duncan
Queen of Blood by Sarah Beth Durst
Vita Nostra by Maryna and Serhiy Dyachenko
How Rory Thorne Destroyed the Multiverse by K. Eason
Malazan (Malazan 1) by Steven Erikson
Daughter of the Empire by Raymond Feist and Janny Wurts
Mistress of the Empire by Raymond Feist and Janny Wurts
Servant of the Empire by Raymond Feist and Janny Wurts
Dragonâs Egg (Cheela 1) by Robert L Forward
Mother of Learning by Domagoj Kurmaic/nobody103
Books of Magic by Neil Gaiman
The Midnight Library by Matt Haig
The Warehouse by Rob Hart
Forging Hephestus by Drew Hayes
Super Powereds, v1 by Drew Hayes
Super Powereds, v2 by Drew Hayes
Super Powereds, v3 by Drew Hayes
Super Powereds, v4 by Drew Hayes
Johannes Cabal by Johnathan L. Howard
The Medusa Plague by Mary Kirchoff
Six Wakes by Muir Lafferty
King of Thorns by Mark Lawrence
Emperor of Thorns by Mark Lawrence
First Contacts by Murray Leinster
Futurological Congress by Stanislaw Lem
Perfect Vacuum by Stanislaw Lem
Tuf Voyaging by George R R Martin
Memory of Empire by Arkady Martine
A Desolation Called Peace by Arkady Martine
Middlegame by Seanan McGuire
The Host by Stephanie Meyers
The city & the city by China Mieville
*The House that Made the 16 Loops of time by Tamsyn Muir
Harrow the Ninth by Tamsyn Muir
Convenience Store Woman by Sayaka Murata
A Deadly Education by Naomi Novik
The Last Graduate (Schoolomance 2) by Naomi Novik
Stiletto (Chequey, book 2) by Daniel OâMalley
Special Topics in Calamity Physics by Marisha Pessl
Carpe Jugulum by Terry Pratchett
Guards! Guards! by Terry Pratchett
Jingo by Terry Pratchett
The Last Continent by Terry Pratchett
Monsterous Regiment by Terry Pratchett
Men at Arms by Terry Pratchett
Night Watch by Terry Pratchett
Snuff by Terry Pratchett
Sourcery by Terry Pratchett
The Truth by Terry Pratchett
The Woven Ring (Solâs Harvest 1) by M D Presley
Years of Rice + Salt by Kim Stanley Robinson
The Torch That Ignites the Stars by Andrew Rowe
Sleep Donation by Karen Russell
A Darker Shade of Magic by V E Schwab
Invisible Life of Addie LaRue by V E Schwab
Vicious by V E Schwab
Vengeance by V E Schwab
Grasshopper Jungle by Andrew Smith
Why Is This Night Different Than All Other Nights? by Lemony Snicket
Dark Storm (Rhenwars 1) by M L Spenser
Anathem by Neal Stephenson
Cryptonomicon by Neal Stephenson
Nimona by Noele Stevenson
Hunter x Hunter manga v1-36 by Yoshihiro Togashi
Worth the Candle by Alexander Wales
Educated by Tara Westover
Soulsmith (Cradle 2) by Will Wight
Blackflame (Cradle 3) by Will Wight
Skysworn (Cradle 4) by Will Wight
Ghostwater (Cradle 5) by Will Wight
Underlord (Cradle 6) by Will Wight
Uncrowned (Cradle 7) by Will Wight
Wintersteel (Cradle 8) by Will Wight
Bloodlines (Cradle 9) by Will Wight
Reaper (Cradle 10) by Will Wight
The Crimson Vault (Travelers Gate 2) by Will Wight
*Dinosaurs by Walter Jon Williams
Blind Lake by Robert Charles Wilson
Thousand Li by Tao Wong
Thousand Li 2 by Tao Wong
Thousand Li 3 by Tao Wong
Thousand Li 4 by Tao Wong
Thousand Li 5 by Tao Wong
Sorcererâs Legacy by Janny Wurts (see also Feist)
Heretical Edge by ceruleuanscrawling
Mark of the Fool by UnstoppableJuggernaut
there is no antimemetics division by qntm
Only Villains Do That by Webbonomicon
Worm by wildbow

Nonfiction:

Compiling with Continuations by Andrew W. Appel
The Rule of Benedict by St Benedict (read the front material only)
Programming Pearls by Jon Bentley
Whole Brain Emulation Roadmap by Nick Bostrom
Data Matching by Peter Christen
Attack and Defense by James Davies and Akira Ishida
Engines of Creation by K. Eric Drexler
Class by Paul Fussell
The Food Lab by J Kenzi Lopez-Alt
Primitive Technology by John Plant
Monero whitepaper by Nicolas van Saberhagen
Secrets and Lies by Bruce Schneier
The Cuckooâs Egg by Clifford Stoll

Hereâs a list of books I read in 2022. The ones in bold I recommend.

Fiction:

1632 by Eric Flint
Alex Verus 1: Fated by Benedict Jacka
Alex Verus 2: Cursed by Benedict Jacka
Alex Verus 3: Taken by Benedict Jacka
Alex Verus 4: Chosen by Benedict Jacka
Alex Verus 5: Hidden by Benedict Jacka
Alex Verus 6: Burned by Benedict Jacka
Alex Verus 7: Bound by Benedict Jacka
Alex Verus 8: Marked by Benedict Jacka
Alex Verus 9: Fallen by Benedict Jacka
Alex Verus 10: Forged by Benedict Jacka
Alex Verus 11: Risen by Benedict Jacka
Art of the Adept 2: Secrets and Spellcraft by Michael G Manning
Art of the Adept 3: Scholar of Magic by Michael G Manning
Aspects by John M Ford
Aurora by Kim Stanley Robinson
Aurora Rising by Amie Kaufman
Bastion (Immortal Great Souls 1) by Phil Tucker
Children of Time by Adrian Tchaikovsky
Citadel: Training in Necessity by Unillustrated
City of Broken Magic by Mirah Bolender
Cradle 11: Dreadgod by Will Wight
Crown of Vengeance by James Mallory and Mercedes Lackey
Cytonic by Brandon Sanderson
Elder Race by Adrian Tchaikovsky
Gamechanger by L. X. Beckett
Genius by Leopoldo Gout
Good Omens by Neil Gaiman and Terry Pratchett
Grand Game by Tom Elliot (LitRPG)
Project Hail Mary by Andy Weir
Head-on by John Scalzi
He Who Fights with Monsters 1 by Shirtaloon
He Who Fights with Monsters 2 by Shirtaloon
He Who Fights with Monsters 3 by Shirtaloon
He Who Fights with Monsters 4 by Shirtaloon
He Who Fights with Monsters 5 by Shirtaloon
Highfire by Eoin Colfer
Immortality Code by Douglas E Richards
In Other Lands by Sarah Rees
Insane City by Dave Barry
Iron Prince by Bryce OâConner and Luke Chmilenko
Isolate (Grand Illusion 1) by L E Modesitt Jr
The Kevin Jenkins Experience by Hambone
Kusuriya no Hitorigoto / Alchemistâs Journal by Natsu Hyuuga et al
The Left-handed Booksellers of London by Garth Nix
Lock In by John Scalzi
Mageâs Blood by David Hair
Mark of the Fool by J M Clarke
Martian Abroad by Carrie Vaughn
Master Li and Number Ten Ox 1: Bridge of Birds by Barry Hughart
Master Li and Number Ten Ox 2: Story of the Stone by Barry Hughart
Master Li and Number Ten Ox 3: Eight Skilled Gentlemen by Barry Hughart
Mazer in Prison by Orson Scot Card
Memory of Earth by Orson Scott Card
Memory of Earth 2: Call of Earth by Orson Scott Card
Millenial Mage by J L Mullins
Neverworld Wake by Marisha Pessl
Orc on the Wild Side by Tom Holt
Pact by wildbow
Penricâs Progress by Louis McMaster Bujold
Penricâs Travels by Louis McMaster Bujold
Perfect State by Brandon Sanderson
Powder Mage 1 by Brian McClellan
Primal Hunter by Zogarth
Quantum Shadows by L E Modesitt (in the style of Gene Wolf)
Ready Player Two by Ernest Cline
Recluse 1: Magic of Recluse by L E Modesitt Jr
Recluse 2: Towers of the Sunset by L E Modesitt Jr
Recluse 3: Magic Engineer by L E Modesitt Jr
Red Rising by Pierce Brown
Red Rising 2: Golden Son by Pierce Brown
Red Rising 3: Morning Star by Pierce Brown
Remnant Population by Elizabeth Moon
RE: Trailer Trash by FortySixtyFour
Revelation Space by Alastair Reynolds
Rook and Rose 1: Mask of Mirrors by M A Carrick
Rook and Rose 2: The Liarâs Knot by M A Carrick
Salvaged by Madeleine Roux
Salvos by V A Lewis (LitRPG)
Scardown by Elizabeth Bear
Servant Mage by Kate Elliot
Significant Digits by Alexander Deebus
Sleep In a Sea of Stars by Chistopher Paolini
Solutions and Other Problems by Allie Brosh
Soulmage by meowcats734
Starsight by Brandon Sanderson
Story of My Life by Hellen Keller
Strange and Stubborn Endurance by Foz Meadows
A Succession of Bad Days by Graydon Saunders
The Starless Sea by Eric Morgenstern
Termination Shock by Neal Stephenson
The Consuming Fire by John Scalzi
The Every by Dave Eggers
The Last Emperox by John Scalzi
The Philosopherâs War by Tom Miller
The Prefect by Alastair Reynolds
The Testing by Joelle Charbonneau
The Truth and Other Stories by Stanislaw Lem
The Unspoken Name by A.K. Larkwood
Thiefâs Magic by Trudi Canavan
Three Body Problem 2: Wallfacer: Dark Forest by Cixin Liu
Throne of the Five Winds by S C Emmett
Under the Pendulum Sun by Jeanette Ng
Venemous Lumpsucker by Ned Beauman
Vigor Mortis by Natalie Maher
Ward by Wildbow
Weirkey 1: Soulhome by Sarah Lin
Weirkey 2: Rainhorn by Sarah Lin
Winterâs Orbit by Everina Maxwell

Nonfiction:

The Art of Computer Programming v1 by Donald Knuth
The Art of Computer Programming v2 by Donald Knuth
Attack and Defense by James Davies
Burning Wheel (RPG) by Luke Crane
The Economist (magazine)
Home Improvement 1-2-3
Illustrated Guide to Everything Sold in Hardware Stores (1988) by Steve Ettlinger
Inadequate Equilibria by Eliezer Yudkowsky
The Prince by Niccolo Machiavelli
Programming Crystal by Ivo Balbaert
Sigbovik 2021
Spymistress by William Stevenson
What If? by Randall Munroe
What If? 2 by Randall Munroe

Hereâs what happened in 2022 for me!

Move

I moved from California to Ohio. I wanted to be with my friends. Also, my old place caught on fire (twice). The new place is cheap, but underground. The first order of business was installing lots of lights, and replacing my moldy old mattress.

My dad kindly lent me a car until mine showed up in September. There was lots of DMV paperwork. Not the best, but Ohio is much cheaper and easier than California in this regard. I also got health insurance, which cost almost as much as my rent.

Happily I already knew many people where I was moving, and I also started attending several meetups from meetup.com. I got to spend Thanksgiving and Christmas with friends and family this year. I also sent out Christmas cards for the first time.

Games

I started two D&D games in 2022. One ended before session 1, the other exploded after two months. I had a nice time playing as a player in âIndex Card RPGâ, though. I ran a session of lexicon, which went pretty well. We quit before getting to the letter Z, but thatâs a design flaw in Lexiconâitâs way too long.

I participated in the 2022 April Fools Puzzle Contest, on #ircpuzzles. I came in 7th.

This and that

A little travel. I went to Missouri to visit friends. I got to go to my friendsâ wedding in Boston.

I read âThe Art of Computer Programmingâ volumes 1 and 2. Donald Knuth sent me a check for finding a 0x1.2 bugs.

I got a snakebite lip piercing.

I made a first-aid kit, which Iâm realizing I didnât write up. My thinking was that itâs bad to give medical advice when you donât know anything about medicine.

I made a new blast furnace with my sister, which we never used (old one).

I made an e-ink laptop.

Software

In November, I did Hack-a-Day, a project I conceived to do a new computer project every day of the month that I could show off to others. As part of it, I learned web sockets, webRTC, unity3D, game programming. In all, there were around 30 projectsâclick the link to see them all.

I made huge improvements to qr-backup. Its basically âdoneâ for the CLI version.
I wrote youtube-autodl, a program to automatically download a feed of youtube videos and sort them into folders.
I wrote a video linter for my personal video collection.
I wrote a screenshotter, which takes one screenshot a minute of my laptop (encrypted) and archives them indefinitely.
I re-wrote is rick and morty out.com for Season 6.
I wrote record-shell and installed it on my computers. It records all shell sessions, etc including ssh sessions.
I wrote Doodle RPG, which I was quite proud of. It ran for a good while and tapered off. It supports mobile!
I did a couple late hack-a-day followups: a bug reporter and hack-a-spring (unfinished).
Worked on beggar-my-neighbor solver.

Habits

I was exercising daily. I kind lapsed after my ankle surgery, oops.

I stopped doing my daily morning log at some point, and didnât fix it within 2022.

I tried an experiment with âno-computerâ sundays. This was super productive one time, and less so the next. It led to the e-ink laptop, because writing a short story by hand was really painful.

I started limiting myself to one youtube video per day. That went great and Iâve kept it up.

Every year I have a checklist of things to do. I did them. Two of the more well-known are my storage cost survey and my media longevity test.

I sorted my scans into folders. I decided not to do the whole process (transcribe the handwritten documents, etc) for the thousands of scans, because it wouldnât be worth the time. Iâll wait and see what I can do with AI in a few years, maybe.

Writing

You can read most of what I wrote here! On a blog! Of particular interest might be my new index page.

I also wrote a short story, Earth II. Itâs not online because itâs bad.

I had to remove library.za3k.com because of DMCAs.

Year 0Â â I filled 10 32-GB Kingston flash drives with random data.
Year 1Â â Tested drive 1, zero bit rot. Re-wrote drive 1 with the same data.
Year 2Â â Tested drive 2, zero bit rot. Re-tested drive 1, zero bit rot. Re-wrote drives 1-2 with the same data.
Year 3 â Tested drive 3, zero bit rot. Re-tested drives 1-2, zero bit rot. Re-wrote drives 1-3 with the same data.

This year they were stored in a box on my shelf.

Will report back in 1 more year when I test the fourth ð

Iâve added a central games page https://za3k.com/mygames.md to my website, with all the games I designed. The new games:

Loot Boxes. Untested. Easy storytelling game for 2-4 players. The players have an inventory of absurd random items, and must solve challenges using each item in turn.

Stupid Russia. Tested. Party game for 10+ people. Each player is a spy director at the Stupid KGB, and must report as many codenames to the Inspector as possible, swapping secret information with other players. The players had fun, especially adopting bad accents. The rules were too hard to understand, and it was too much work and no fun for me as the Inspector. Overall Iâd just recommend Stupid Conspiracies instead.

Stupid Conspiracies. Untested. Party Game for 8+ people. Each player tries to recruit the others into their conspiracy, for about half an hour. Itâs a re-write of the core idea in Stupid Russia. Overall, big party games are just too hard for me to organize.

I also playtested âNo this cannot be! I AM INVINCIBLE!â. It ran about 45 minutes prep (not fun) and 45 minutes playtime, which was the main problem. Overall the play time was fun. I rewrote it to have MUCH easier prep, and for the game to be generally easier. I also re-wrote the rules of âNinjas Ninjas Ninjasâ without a playtest. I donât think it will ever be too popular but it has a soft spot for me.

Emperical Zendo, a semi-competitive game for 3-8 players based on the icehouse game Zendo. Vaguely based on rants by Bayesians.
Logic Potions, a competitive game about deductive logic and making new rules for 2-4 players. Actual gameplay quickly gets complicated as players add more rules about brewing potions. Inspired by âImaginary Go Fishâ and âEmperical Zendoâ.
Deadly Education RPG, a traditional pen+paper RPG game based on Naomi Novikâs âDeadly Educationâ. Reading the book is not required.

All three are untested as of posting.

Always use clock time.

Yes, there are interruptions. You need your coffee. You didnât get around to it that day. You want to know those things in your estimate, too. Just use the time on the clock for when a task starts and ends.

This is especially important if youâre self-employed.

Write down how long you think a task will take. Afterwords, write down how long it took.

This simple step is the most important one. This gives you a clear idea of exactly what a task is and when itâs done. It also starts automatically training your brain.

Youâll start seeing patterns. You consistently underestimate how long everything will take. Conversations take longer than they feel. Exercise takes less time than it feels like. Fixing problems is highly variable. Doing something from scratch is easier to predict.

Play a game. Predict things as well as possible.

Donât change how you do them. You win if you guess accurately.

Use as few units as possible.

Donât use minutes, hours, days, weeks, and months. Personally, I try to use minutes and hours for everything. Of course, when I report to my boss, I convert to days, but in my own notes I estimate things in one unit.

Learn your multiplication factor.

How long will it take you to do a project? Well, last time you had a similarly-sized project, you thought it would take 2 hours, and it actually took 14 hours. Your multiplication factor is about 7x. So this time if it feels like a 3 hour task, plan for 21 hours.

Assume thereâs only one multiplication factor for one kind of work (one kind of work like your entire job, not one type of task). You can have different ones for different time scales, though (minutes vs hours vs days vs weeks).

You can measure other peoplesâ multiplication factor to figure out when theyâll actually be done with tasks, but I suggest doing it quietly and not mentioning it.

Credit: Folk, but credit to Joel on Software for the idea of estimating it for each team member

To estimate a long task, break it up into pieces, and add up the pieces.

Do this if your task takes 2 days or more. Because of the multiplication factor, carefully budget time for added tasks, things you forgot, problems, etc. Or you can skip it. Just consistently pick one.

Credit: Joel on Software, including FogBugz which did this as a statistical method displayed with Gantt Charts.

Train your credible intervals.

Some tasks are more variable. Saying âsomething will take 1 hourâ is vague. Saying âsomething will almost certainly take between 30 minutes and 4 hoursâ is more precise. How big should that range be? Thatâs called a credible interval.

Train your credible intervals. I trained mine using bug fixing, something which happens several times a day, is hard to predict, and you have little control over (you canât âcall it doneâ early). Customer calls could be another great candidate.

I trained on bugfixes using 50%, 90%, and 99% intervals. There are specific mathematical scoring rules, but basically if something is in your 50% interval more than half the time, narrow it; if your interval is correct less than half the time, widen it.

Credit: Eliezer Yudkowsky (personal website, no longer up)

-
HACK (noun)
-
-
a rough cut, blow, or stroke. (the work was accomplished one hack at a time)
-
a quick job that produces what is needed, but not well (this code is a hack, but it works!)
-
-

Hack-A-Day is challenge to make complete one new project, from scratch, every day in November 2023.

Last year (2022), I set myself the challenge to make a software project every day, and met it. I had a ton of fun, and make a lot ofÂ cool video games and projectsÂ I can show off. This year Iâm inviting the rest of the world to join me!

Iâm a programmer, so Iâm doing a new computer programming project every day. But you can do any kind of project, whatever you pick is great.

I encourage you to join. I would guess this takes 2-4 hours a day (similar to NaNoWriMo). But if you donât have that kind of time, please do still join for as many days as you can! And if you want to collaborate with me, set aside a free day and message me by email. My calendar is at zachary.youcanbook.me. Feel free to grab any day starting the 4th!

Hack-A-Day 2023 is complete. I did 20 projects in 30 days. Pretty good considering I got a new job and moved!

The overview of the month is here and highly recommended.

I plan to do a little more followup this year than last. Todayâs last-minute fixes:

Perquackey (01) works on phones, tablets, and with the mouse. Thereâs an easter egg that displays any words you missed at the end.
Typewriter (10) saves progress, and lets you type more than 1 page of content.
Screensaver (12) works on more screen sizes, including phones.
Synth (14) looks better on a phone-sized display.
Stuff.md (23) has an example from the database
Timelapse (24) has a partial video
Speed Reading (29) works better on a phone. It also saves your progress.
Music of the Spheres (30) works on a phone. Itâs also louder.

-
HACK (noun)
-
-
a rough cut, blow, or stroke. (the work was accomplished one hack at a time)
-
a quick job that produces what is needed, but not well (this code is a hack, but it works!)
-
-

Hack-A-Day is challenge to make complete one new project, from scratch, every day in November 2023.

Iâm a programmer, so Iâm doing a new computer programming project every day. But you can do any kind of project, whatever you pick is great.

I will post again nearer to November! Just giving blog readers an advance heads-up.

Today I chose to write a web version of a word game my family has loved for a long time, but which is sadly out of print.

It is meant to be played multi-player, but youâre welcome to try it out single-player online. Have fun!

Source code here

Lashed furniture is made using sticks and rope or twine. Todayâs project was to make one out of bamboo and brown paracord. The frame is shownâimagine boards or many pieces of bamboo forming a top.

Hello to the young lady who decided to pose and join in the photo!

We found this little $5 tool to be incredibly good for cutting bamboo. Itâs designed for almost the same thing, cutting metal pipes.

It wasnât bad for a first try. That said, we decided the top wasnât flat enough to give a good finish, so the whole thing is going to be burned at the next bonfire.

-
-
-Carin says: -
-
- November 6, 2023 at 8:44 am
-
-
Cool!
-
-
-Reply
-
-

I made a small game called Doodlemoji Alchemy, together with my friend Jennifer, as part ofÂ Hack-A-Day.

You can play itÂ here.

You combine elements to make something new. Sometimes you get an old element:

Sometimes you discover a new one!

Todayâs update is a short one. I ported my raytracer from day 02, to the Nvidia GPU: ha3k-06-raytracer

The visuals are pretty much the same. Incidentally I discovered the striations on the ground disappear if we increase the floating point precision.

Render on the GPU is 30x faster (0.05 fps -> 3 fps). Thatâs still not very fast.

~~I didnât get video working yesterday, or anything else visually new. I will call this one a failure overall, because I have nothing interesting to show off.~~ I learned stuff and made progress though, so itâs not so bad.

Hereâs a working video!

- -

Iâve wanted to make a receipt-printer zine for a while. Hereâs the next best thing, a digital verson.

Click the image for a silly little zine.

Try it out here. Code is on github.

A simple screensaver made in my raytracer. Code is on github.

- -

A âsilly screensaverâ. Demo is here. Source code is on github.

Hack-A-Day is a challenge to try and finish 30 projects in 30 days in November.

Today I tried to write a tool to make a floorplan. You can try it here. As usual the source code is on github.

This was an ambitious project for one day, and I didnât finish everything I wanted. My original goal was to support

drawing and erasing rectangles (done)
adding, editing, deleting, and moving text labels (not done)
adding, deleting, and moving icons (mostly not done)
autosave (done)
undo support (done)
zoom and pan (not done)
sharing finished projects (stretch goal, not done)

What I did do was pleasantly high-quality, and I made pretty good progress.

My friend Kragen and I wrote a little bytebeat synth tool. You can mess around and have fun. Demo here, code is on github.

Todayâs hack-a-day project was a pencil-and-paper RPG. Based on feedback from people reading the rules, itâs notably bad and I donât recommend it. Rules here.

Today I got a variety of modern A.I. tools to work in a python library. This one is mostly install instructions, but it was useful for me, at least.

I took a day off after.

My conclusions were:

Speech synthesis runs at 400x realtime on CPU.
Speech recognition runs at 0.4x realtime on CPU, 60x realtime on GPU.
Generating chat at 0.05x â 0.5x realtime (3-30 wpm) on GPU.

I didnât get image generation working on my allotted time.

Today I wrote a simple raytracer. A raytracer is a very simple way to draw excellent graphics. For each pixel, it follows an imaginary âlineâ out from the viewer through that pixel into the computer world. Then it colors the pixel based on what the line hits. Unfortunately, it also takes a lot of computing power.

Mine is based on the explanation (and code) from âRay Tracing in One Weekendâ, and the code from âMy Very First Raytracerâ.

Matte spheres in different shades of grey. The blue in the spheres is reflected from the sky.

The motivation for this project was to learn how to make things run faster on a graphics card. I quickly realized (before I wrote a line of code) that Iâd need the basic raytracer to be its own project. Having it run faster will have to be a job for another day!

A final demo scene, showing off reflectivity and metal surfaces.
Note the pincushion distortion of the overall render, and striations on the ground.

For todayâs hack-a-day, I meant to clone the Hillsfar lockpicking minigame. Instead, I spent all day just extracting the sprites. But I had a nice chill time, so it was great.

Edit: See the updated post for the finished game.

Hereâs the original minigame:

Hereâs my spritesheet:

I made it by splitting up screenshots:

Today I learned how to make PCBs. I didnât invent anything here, this is all pretty well known by the PCB-making community, but itâs not well-known to me. So I taught myself a bit!

The first part was the design an electronic circuit. I decided I was short on time, so I grabbed an existing schematic.

Next, I downloaded KiCAD, and recreated the circuit there. I found this video tutorial very helpful to learn kicad.

Next, I made the actual PCB layout.

To my surprise, after a little jiggling I got it down to a one-layer design.

That means home-printing would be much easier. No having to line up the two sides carefully.

I printed out the image on paper (backwards) on my toner printer, and taped it to the copper-clad PCBs.

First, I tried laminating it. Almost no ink transferred, and the paper came off easily. Then I tried ironing it, but the paper stick to the iron and not to the PCB. The tape melted on the iron. For both, I dunked them in water after, which is supposed to help loosen the paper.

Next, I tried the standard adviceâsand the PCBs (I used 320 grit) and use glossy paper. This time, both pieces of paper stuck very well. I was wary about the iron coming off again, so I just left it on place on the highest heatâthis worked fine for adhesion, but I had to iron out wrinkles at the end. The laminated piece had lose edges, while the ironed piece was on there totally flat.

I tried peeling off the laminated paperâoops! It peeled back and most of the ink stayed on the paper. I think if I took it off more carefully, it would have worked.

I picked at the ironed paper a bit, but it didnât budge. I let it sit in dish soap for a while so the paper would fall apart. The first hour didnât do anything.

Meanwhile, I made an order at PCBWay. Itâs still under review.

Edit: after some advice from a friend, I peeled off this paper more aggressively, and scrubbed it off. The ink was fine. It doesnât look great, but I think this is mostly the wrinkles during transfer. Itâs a little blurry, Iâll have to do a third attempt before I try etching.

Iâm moving, so I have to pack. I thought Iâd make it fun with two projects.

First, I entered everything I was packing into a text file, stuff.md. That way, I can find stuff later. I have two friends who have done something like this, so Iâm curious how it will go for me. Here is a sample:

Box 01 - banker
-======
-- USB Receipt Printer - in trapezoid box
-- Thinkpad 460 Charger (x2) - cardboard box (x2)
-- Cardboard box "eink"
-    - eink communications converter for 7.5" eink display
-    - Piece of fiberglass sized for 7.5" eink display
-    - 1.54" eink display 152z152px never used, with notes on yellow paper
-- Airtec electric duster (AC) - cardboard box
-- Tiny UPS for Raspberry Pi - cardboard box
-- Wireless receipt printer - cardboard box
-- Playstation Eye (x2) - cardboard box
-- Mini-router (2 eth, 1 usb), unconfigured - in cardboard box
-- Pipe-sealing tape for vacuum - loose
-- Multimeter, Kaiweets brand - in cloth case
-- $1 in pennies, and penny sleeves - plastic bag
-- Engraving pen - loose metal case
-- LED Light bulbs (one white, one red) - cardboard box
-
-Box 02 - banker
-======
-- HDD Copier - cardboard box
-- HDD Dock (x2) - cardboard box (x2)
-- Butane soldering iron - metal box
-- Doxie Go adapters - loose plastic bag
-- "Faces" M5Stack development. Stacking keyboard and screen, etc. - plastic case
-+ Magnetic metal parts tray
-+ Neodynium magnets, two disc sizes - Loose box
-+ Receipt paper roll - loose
-
-Box 03 - banker
-======
-- empty
-

Second, I took a time lapse video of packing. I wish I had time-lapsed moving in at my current place, but I just wasnât set up for it. Sadly, my camera battery died after 90 minutes, so I only have a very short video. Next time Iâll plug in a power cable. Here is a short example video.

Both are much too personal for me to post on the web in full.

Two friends and I wrote the intro to âPint-Sizedâ, a 90s sitcom that never existed.

- -

We used DALL-E and stable diffusion for images, Photopea to add captions, and Googleâs AI Test Kitchen for the backing music. Cheers were added with audacity. The video was edited together with ffmpeg.

Credits: za3k, stetson blake, jeremy mcintyre

Yesterdayâs project was Speed Reading. Experience what itâs like to read Don Quixote faster than youâre comfortable with. Source is on github as usual.

Hack-a-Day is a challenge to complete ~30 fun new projects in 30 days. In my case, I aimed for 20, because I knew I was getting a job and moving. I just barely made it with this last entry, a collaboration with nsh.

Music of the Spheres lets you hear songs on different tonal scales. Listen to the warped melodies. Watch the pretty planets orbit. Surely their sizes and orbits are significant and connected to the tonal scales? Go mad with afterimages ofâ¦ okay, well itâs kinda fun, anyway. Demo is here, code is on github.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Yesterdayâs project was Hack-A-Battle (demo, source). Itâs two dueling music visualizers (sound warning!). Red vs blue. As each hits the other with bullets, they lose heath. As a band takes damage, it gets dimmer and quieter. Eventually one band will win out and be the only one playing.

I thought this was a cool idea, but Iâm not really happy with the implementation

Itâs a little laggy, especially when explosions happen.
Itâs probably a little too fast of a battle.
I wanted to the things coming out to actually be linked to a music visualizer, which I almost had time to do.
It would have been better if the âbandsâ took turns playing instead of both going at once, for the poor listener.
It requires a fairly big display, and beefy computer/phone. It doesnât work well on a small screen at all.
I wasnât super pleased with the code. It was so-so
I wanted you to be able to upload your own songs and duel a friend

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is the Hack-A-Blog. (demo, source).

Check out the link above to try out the live demo. Iâm proud of getting this one done in time. I think the next days will be easier, as I figured some things out already.

Iâm continuing Hack-A-Day, I think. Todayâs project is Hack-A-Bug (demo, source). Itâs a bug reporter I can add with one line to any of my projects.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Chat. (demo, source). Itâs a free web chat for anyone that goes to the website.

Check out the link above to try out the live demo.

So far, not much easier. Another 8-hour day. I was hoping to do something with WebRTC today, but I didnât get to it.

Thursdayâs project was Hack-A-Clock (demo, source). It is a decimal time clock, displaying the time in revolutionary french time (minus their weird calendar).

This is another âphone it inâ project but I think it would have been okay with more accompanying explanation and better styling.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Crop (demo, source). It crops an image to a fixed size.

I got help from several people on the CSS, thanks to instantly sharing my work via ngrok from my laptop. Thanks people! Thanks ngrok!

Todayâs project was in anticipation of showing off what I did at the end of the month. I want a small thumbnail for each project.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Dictionary (demo, source). It looks up words.

OK, Iâll be honest. Iâm phoning this one in. I needed a break.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Farm (demo, source). Itâs a simple tile-based RPG. You can walk around as a chicken, admire your house, and plant and harvest two types of crops.

My main goal with this project was to work with spritesheets or animation before, which I had never done. Showing off the individual tiles is deliberate. Also, the game should respond well to smaller and larger screens, I hope.

I had a good time with this one, and Iâm happy with how much I got done in a day. I originally planned to do more fluid walking (it was called Hack-A-Walk), but it was more fun to add crops instead.

I re-used some of the logic from Hack-A-Minigame and Hack-A-Snake. Iâve been finding d3 to be mildly useful, if a little annoying.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Hang (demo, source). Itâs a place to hang out. It has text chat, video, and audio.

Hack-A-Hang is NOT WORKING.

Unfortunately while everything works great on my machine, thereâs a bad problem in production, and I ran out of time on this one. Iâll try to get it fixed before the end of the month if itâs easy.

Hoo boy, this was one of the technically hardest ones so far. WebRTC is no joke. And not hard in a way where you have to think, hard in a way where the debugging tools are terrible. (Drag and drop was another tough one)

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Hell (demo, source). Itâs a bullet hell game combined with a music visualizer.

Iâm happy with this one, although it took way too long given yesterdayâs project! I keep thinking Iâll be able to modify or re-use things quickly, and itâs not true.

P.S. Taking the next day or two off for thanksgiving

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs (catch-up) project is Hack-A-Homepage (demo, source). You can enter various information about yourself, such as links to your social media, and make your own little homepage.

This one took about another hour. I think itâs okay, but today was definitely a âdo the numbersâ game to catch up. Tomorrow I want to do something more fun and new.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Line (demo, source). Hack-A-Line is a 5-in-a-row game for two players. You play online against each other by sharing a link.

Iâm okay with this one, except that thereâs one really bad display bug that kind of ruins it. Iâm starting to develop a list of projects where I might want to go back and fix something after this month.

I skipped yesterday because I needed a break.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Wednesdayâs project was Hack-A-Link 2 (demo, source). Itâs an open link directory.

This was very much a âphone it inâ project. I do think it could have been better if I had added descriptions, too.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs (catch-up) project is Hack-A-Link (demo, source). You can enter a long link, and it will be shortened.

This one was also pretty easy. About an hour.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Machine (demo, source). Itâs a whimsical VM you can play around with.

I liked this one, but it really didnât fit into a day too well.

Features include:

Edit, save, and share programs!
Visual debugger with step, run, pause, and stop!
One example program!
Weird arithmetic!
Negative-364 based addressing!
âTouchscreenâ display!
âTicker tapeâ input and output!
21 never before seen instructions!
Optional registration!

Sadly not in the âone dayâ category are

good compiler errors
line numbering
line highlighting when stepping through code
syntax highlighting
the entire machine 100% working (I tested a little!)
fixing all the bugs (thereâs a particularly nasty one where you have to save before you run)

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Mandelbrot (demo, source). Interactively explore the fractal world of the Mandelbrot set.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Minigame (demo, source). Itâs the classic Snake, but the twist is you can only save and load the game. Rather than controlling the snake, it moves at random under AI control. You have to repeatedly save and load to make progress.

Credit to Jeff Laitâs âSave Scummerâ 7-day roguelike for inspiration. Although actually, this whole minigame is mostly for a future project!

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Paste (demo, source). You can upload text and share it with others.

Todayâs project was an easy one, because I was busy. Took an hour or two.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Snake (demo, source). Yesterday I wrote a game where an AI plays snake. Today I thought, hey, I should release that with keyboard controls so people can just play Snake.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Song (demo, source). It plays greensleeves, and you can edit the music to play something else, too.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Sound (demo, source). Itâs a small soundboard.

You can:

Play 25 built-in samples
Record your own samples with your mic
Play samples directly, or record them on up to 4 tracks
Have tracks play one time only, or loop
Adjust the alignment of tracks
Use a keyboard or mouse
(Mobile not supported)

Have fun. I think this was a pretty good toy.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Stats (demo, source). It displays web traffic statistics about Hack-A-Day.

The original point of the project was to show some nice graphs in d3, as an effort to teach myself d3. But halfway through I got unbearably bored by âshow a bunch of statsâ as a project, so no d3. Whoops! You win some you lose some.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Tank (demo, source). Itâs a relaxing fishtank toy. Enjoy.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-A-Tile (demo, source). Itâs a tile-matching game like dominos.

Hack-A-Tile is based on mathematical Wang tiles. It was very tempting to call it Hack-A-Wang.

If I update it, I would

Zoom out as you go. I think that would look cool!
Animate shifting over. Right now adding tiles on the top or left is visually confusing.
Change the tiles. These are fun mathematically, but not ideal for a game
Either add a maximum size, or some constraint to stop you just making one long line.

Itâs no longer november, but Iâm still doing a project a day. Itâs an all-month hack-a-thon!

Yesterdayâs project was Hack-A-TV-Guide (demo, source). Itâs a TV Guide generated from Wikipedia. I got the idea from having written isrickandmortyout.com. Why not do the same thing, but for every show?

Iâm going to call this one a flop. Thereâs a good version of this project, but I ran out of time. Basically all it does is display info about a show, which is not very useful.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-An-Adventure (demo, source). Itâs a coloring book. Itâs designed to be relaxing.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-An-Asteroid (demo, source). Itâs an asteroids clone with four levels.

Iâm pretty happy with this as my first âvisualâ game. I made it in Unity3D.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project was Hack-An-Experiment (demo, source). Itâs designed to present the basics of experimental algorithmics, while also getting me acquainted with d3.

I have to say, I keep seeing d3 sold as a âgraphingâ library. And itâs definitely not. Maybe you could write one on top of it.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

Todayâs project is Hack-An-Icecube (demo, source). Itâs a failed attempt to make a game in Unity3D, the game engine/framework.

This is my first failure, which I think is good! I was getting bored only attempting things I could definitely succeed at in one day. That said, running out of time is fun. Wrestling the IDE isnât. This was mostly wrestling the IDE, setting configs, and installing software.

I wonât continue this tomorrow, Iâd rather do a new project each day (Itâs called Hack-A-DAY)! But I might do another Unity3D project before the month is out.

The ice cubes load and fall, and thatâs it. No controls, no game. I ran out of time.

This is an attempt to recreate One Hour Jam Cannon by juzek exactly. I ran out of time, but I made some progress getting Unity to work.

Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon!

This is November 30th, so this will be the last project.

Todayâs project is Hack-An-MMO (demo, source). Itâs a small collaborative art RPG. You can draw people, places, and things to populate the tiny world. Have fun!

Itâs no longer november, but Iâm still doing a project a day. Itâs an all-month hack-a-thon!

Todayâs project is Hack-An-Uptime (demo, source). Itâs a simple experiment where you press a button every day.

This was a very simple project. Iâm curious what will happen with it.

Hack-A-Day has its own website now: https://za3k.com/hackaday

Yes, this is why I wrote Hack-A-Crop.

This is my new blog! The old content is archived.

The game can be played here. Source code is on github.

Info about the original 1989 Hillsfar game is linked from the demo! Enjoy.

See also the previous blog post about the spritesheet.

I retired at 31, and get asked about it sometimes. I wrote an article about how the math of retirement, which explains how I retired early (and some some extent, why). And of course, how and why you might want to as well.

I want to edit my finances articles, so this one is on my website instead: https://za3k.com/finance/retire_forever

There will probably be some more finances articles to come soon.

I made an animated HTML + CSS cheatsheet. This took me about three days. It is not really intended for beginners. It contains stuff I frequently forget myself.

(From Stack Overflow)

64 bit requirements for pre-made binaries: -

sudo dpkg --add-architecture i386
-sudo apt-get update
-sudo apt-get install libstdc++6:i386 libxml2:i386 zlib1g:i386 libpopt0:i386

Install CUPS -

sudo apt-get update
-sudo apt-get install cups

Download DriverGo to http://support-au.canon.com.au/contents/AU/EN/0100459602.html and download driver -
```
64e2d00f0c8764d4032687d29e88f06727d88825 Linux_CAPT_PrinterDriver_V270_uk_EN.tar.gz
```
-

Extract and enter extracted folder -

tar xf Linux_CAPT_PrinterDriver_V270_uk_EN.tar.gz
-cd Linux_CAPT_PrinterDriver_V270_uk_EN

Install the custom drivers and ccpd -

sudo dpkg -i 64-bit_Driver/Debian/*.deb
-

Add the printer to CUPS and ccpd -

sudo lpadmin -p CANON_LBP6000 -m CNCUPSLBP6018CAPTS.ppd -v ccp://localhost:59687
-sudo lpadmin -p CANON_LBP6000 -E
-
-sudo ccpdadmin -p CANON_LBP6000 -o /dev/usb/lp0

Set the default printer -
```
sudo lpoptions -d CANON_LBP6000
```
-
Set ccpd to start on boot -
```
sudo update-rc.d ccpd defaults
```
-

Iâm posting my email setup here. The end result will:

Use Postfix for SMTP
Use Dovecot for IMAP and authentication
Store usernames, email forwards, and passwords in a Postgres SQL database
Only be accessible over encrypted channels
Pass all common spam checks
Support SMTP sending and IMAP email checking. I did not include POP3 because I donât use it, but it should be easy to add
NOT add spam filtering or web mail (this article is long enough as it is, maybe in a follow-up)

Note: My set up is pretty standard, exceptÂ that rDNS for smtp.za3k.com resolves to za3k.com because I only have one IP. You may needÂ to change your hostnames if youâre using mail.example.com or smtp.example.com.

On to the install!

Install debian packages -

sudo apt-get install postfix # Postfix \
-      dovecot-core dovecot-imapd dovecot-lmtpd # Dovecot \
-      postgresql dovecot-pgsql postfix-pgsql # Postgres \
-      opendkim opendkim-tools # DKIM

Set up security. smtp.za3k.com cert is at /etc/certs/zak3.com.pem, the key is at /etc/ssl/private/smtp.za3k.com.key. dhparams for postfix are at /etc/postfix/dhparams.pem. (If you need a certificate and donât know how to get one, you can read Setting up SSL certificates using StartSSL)

Install Postfix -

# /etc/postfix/master.cf
-smtp       inet  n       -       -       -       -       smtpd
-submission inet  n       -       -       -       -       smtpd
-  -o syslog_name=postfix/submission
-  -o smtpd_tls_security_level=encrypt
-  -o smtpd_sasl_auth_enable=yes
-  -o smtpd_reject_unlisted_recipient=no
-  -o milter_macro_daemon_name=ORIGINATING
-

# /etc/postfix/main.cf additions
-# TLS parameters
-smtpd_tls_cert_file=/etc/ssl/certs/smtp.za3k.com.pem
-smtpd_tls_key_file=/etc/ssl/private/smtp.za3k.com.key
-smtpd_use_tls=yes
-smtpd_tls_mandatory_protocols=!SSLv2,!SSLv3
-smtp_tls_mandatory_protocols=!SSLv2,!SSLv3
-smtpd_tls_protocols=!SSLv2,!SSLv3
-smtp_tls_protocols=!SSLv2,!SSLv3
-smtpd_tls_exclude_ciphers = aNULL, eNULL, EXPORT, DES, RC4, MD5, PSK, aECDH, EDH-DSS-DES-CBC3-SHA, EDH-RSA-DES-CDC3-SHA, KRB5-DE5, CBC3-SHA
-
-# Relay and recipient settings
-myhostname = za3k.com
-myorigin = /etc/mailname
-mydestination = za3k.com, smtp.za3k.com, localhost.com, localhost
-relayhost =
-mynetworks_style = host
-mailbox_size_limit = 0
-inet_interfaces = all
-smtpd_relay_restrictions = permit_mynetworks,
-  permit_sasl_authenticated,
-  reject_unauth_destination
-
-alias_maps = hash:/etc/aliases
-local_recipient_maps = $alias_maps
-mailbox_transport = lmtp:unix:private/dovecot-lmtp
-

Install Dovecot -

# /etc/dovecot/dovecot.cf
-mail_privileged_group = mail # Local mail
-disable_plaintext_auth = no
-
-protocols = imap
-
-ssl=required
-ssl_cert = </etc/ssl/certs/imap.za3k.com.pem
-ssl_key = </etc/ssl/private/imap.za3k.com.key
-
-# IMAP Folders
-namespace {
- inbox = yes
- mailbox Trash {
- auto = create
- special_use = \Trash
- }
- mailbox Drafts {
- auto = no
- special_use = \Drafts
- }
- mailbox Sent {
- auto = subscribe
- special_use = \Sent
- }
- mailbox Spam {
- auto = subscribe
- special_use = \Junk
- }
-}
-
-# Expunging / deleting mail should FAIL, use the lazy_expunge plugin for this
-namespace {
- prefix = .EXPUNGED/
- hidden = yes
- list = no
- location = maildir:~/expunged
-}
-mail_plugins = $mail_plugins lazy_expunge
-plugin {
- lazy_expunge = .EXPUNGED/
-}
-

# /etc/postfix/main.cf
-# SASL authentication is done through Dovecot to let users relay mail
-smtpd_sasl_type = dovecot
-smtpd_sasl_path = private/auth
-

Set up the database and virtual users. Commands -

# Create the user vmail for storing virtual mail
-# vmail:x:5000:5000::/var/mail/vmail:/usr/bin/nologin
-groupadd -g 5000 vmail
-mkdir /var/mail/vmail
-useradd -M -d /var/mail/vmail --shell=/usr/bin/nologin -u 5000 -g vmail vmail
-chown vmail:vmail /var/mail/vmail
-chmod 700 /var/mail/vmail
-

psql -U postgres
-; Set up the users
-CREATE USER 'postfix' PASSWORD 'XXX';
-CREATE USER 'dovecot' PASSWORD 'XXX';
-
-; Create the database
-CREATE DATABASE email;
-\connect email
-
-; Set up the schema 
-
-CREATE TABLE aliases (
-    alias text NOT NULL,
-    email text NOT NULL
-);
-
-CREATE TABLE users (
-    username text NOT NULL,
-    domain text NOT NULL,
-    created timestamp with time zone DEFAULT now(),
-    password text NOT NULL
-);
-
-REVOKE ALL ON TABLE aliases FROM PUBLIC;
-GRANT ALL ON TABLE aliases TO postfix;
-GRANT ALL ON TABLE aliases TO dovecot;
-
-REVOKE ALL ON TABLE users FROM PUBLIC;
-GRANT ALL ON TABLE users TO dovecot;
-GRANT ALL ON TABLE users TO postfix;
-

# /etc/dovecot/dovecot.conf
-# Since we're giving each virtual user their own directory under /var/mail/vmail, just use that directly and not a subdirectory
-mail_location = maildir:~/
-
-# /etc/dovecot/dovecot-sql.conf defines the DB queries used for authorization
-passdb {
-  driver = sql
-  args = /etc/dovecot/dovecot-sql.conf
-}
-userdb {
-  driver = prefetch
-}
-userdb {
-  driver = sql
-  args = /etc/dovecot/dovecot-sql.conf
-}
-

# /etc/postfix/main.cf
-pgsql:/etc/postfix/pgsql-virtual-aliases.cf
-local_recipient_maps = pgsql:/etc/postfix/pgsql-virtual-mailbox.cf 
-

# /etc/postfix/pgsql-virtual-aliases.cf
-# hosts = localhost
-user = postfix
-password = XXXXXX
-dbname = email
-
-query = SELECT email FROM aliases WHERE alias='%s'
-

# /etc/postfix/pgsql-virtual-mailbox.cf
-# hosts = localhost
-user = postfix
-password = XXXXXX
-dbname = email
-
-query = SELECT concat(username,'@',domain,'/') as email FROM users WHERE username='%s'
-

# /etc/dovecot/dovecot-sql.conf
-driver = pgsql
-connect = host=localhost dbname=email user=dovecot password=XXXXXX
-default_pass_scheme = SHA512
-password_query = SELECT \
-  CONCAT(username,'@',domain) as user, \
-  password, \
-  'vmail' AS userdb_uid, \
-  'vmail' AS userdb_gid, \
-  '/var/mail/vmail/%u' as userdb_home \
-  FROM users \
-  WHERE concat(username,'@',domain) = '%u';
-
-user_query = SELECT username, \
-  CONCAT('maildir:/var/mail/vmail/',username,'@',domain) as mail, \
-  '/var/mail/vmail/%u' as home, \
-  'vmail' as uid, \
-  'vmail' as gid \
-  FROM users \
-  WHERE concat(username,'@',domain) = '%u';
-

Set up users. Example user creation: -

# Generate a password
-$ doveadm pw -s sha512 -r 100
-Enter new password: ...
-Retype new password: ...
-{SHA512}.............................................................==
-

psql -U dovecot -d email
-; Create a user za3k@za3k.com
-mail=# INSERT INTO users (
-    username,
-    domain,
-    password
-) VALUES (
-    'za3k',
-    'za3k.com'
-    '{SHA512}.............................................................==',
-);
-

Set up aliases/redirects. Example redirect creation: -

psql -U dovecot -d email
-; Redirect mail from foo@example.com to bar@example.net
-mail=# INSERT INTO users ( email, alias ) VALUES (
-    'bar@example.net',
-    'foo@example.com'
-);
-

Test setup locally by hand. Try using TELNET.Â Test remote setup using STARTSSL. This is similar to the previous step, but to start the connection use: -
```
openssl s_client -connect smtp.za3k.com:587 -starttls smtp
```
-
Make sure to test email to addresses at your domain or that youâve set up (final destination), and emails youâre trying to send somewhere else (relay email)
-
A small digression: port 25 is used for unencrypted email and support STARTTLS, 587 is used for STARTTLS only, andÂ 465 (obsolete) is used for TLS. My ISP, Comcast, blocks access to port 25 on outgoing traffic.
Make sure youâre not running an open relay atÂ http://mxtoolbox.com/diagnostic.aspx
Set your DNS so that theÂ MX record points at your new mailserver. Youâll probably want a store and forward backup mail server (mine is provided by my registrar). Email should arrive at your mail server from now on. This is the absolute minimum setup. Everything from here on is to help the community combat spam (and you not to get blacklisted).

Set upÂ DKIMÂ (DomainKeys Identified Mail). DKIM signs outgoing mail to show that itâs from your server, which helps you not get flagged as spam.
-None of these files or folders exist to begin with in debian. -

# Add to /etc/opendkim.conf
-KeyTable                /etc/opendkim/KeyTable
-SigningTable            /etc/opendkim/SigningTable
-ExternalIgnoreList      /etc/opendkim/TrustedHosts
-InternalHosts           /etc/opendkim/TrustedHosts
-LogWhy yes
-

# /etc/opendkim/TrustedHosts
-127.0.0.1
-[::1]
-localhost
-za3k.com
-smtp.za3k.com
-

mkdir -p /etc/opendkim/keys/za3k.com
-cd /etc/opendkim/keys/za3k.com
-opendkim-genkey -s default -d za3k.com
-chown opendkim:opendkim default.private
-

# /etc/opendkim/KeyTable
-default._domainkey.za3k.com za3k.com:default:/etc/opendkim/keys/za3k.com/default.private
-

# /etc/opendkim/SigningTable
-za3k.com default._domainkey.za3k.com
-

Display the DNS public key to set in a TXT record with:

# sudo cat /etc/opendkim/keys/za3k.com/default.txt
-default._domainkey      IN      TXT     ( "v=DKIM1; k=rsa; "
-          "p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCggdv3OtQMek/fnu+hRrHYZTUcpUFcSGL/+Sbq+GffR98RCgabx/jjPJo3HmqsB8czaXf7yjO2UiSN/a8Ae6/yu23d7hyTPUDacatEM+2Xc4/zG+eAlAMQOLRJeo3z53sNiq0SmJET6R6yH4HCv9VkuS0TQczkvME5hApft+ZedwIDAQAB" )  ; ----- DKIM
-
-# My registrar doesn't support this syntax so it ends up looking like: 
-$ dig txt default._domainkey.za3k.com txt
-default._domainkey.za3k.com. 10800 IN   TXT     "v=DKIM1\; k=rsa\; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCggdv3OtQMek/fnu+hRrHYZTUcpUFcSGL/+Sbq+GffR98RCgabx/jjPJo3HmqsB8czaXf7yjO2UiSN/a8Ae6/yu23d7hyTPUDacatEM+2Xc4/zG+eAlAMQOLRJeo3z53sNiq0SmJET6R6yH4HCv9VkuS0TQczkvME5hApft+ZedwIDAQAB"
-

# Uncomment in /etc/default/opendkim
-SOCKET="inet:12345@localhost" # listen on loopback on port 12345
-

# /etc/postfix/main.cf
-# DKIM
-milter_default_action = accept
-milter_protocol = 6
-smtpd_milters = inet:localhost:12345
-non_smtpd_milters = inet:localhost:12345
-

Set up SPFÂ (Sender Policy Framework). SPF explains to other services which IPs can send email on your behalf. You can set up whatever policy you like.Â A guide to the syntax is at: http://www.openspf.org/SPF_Record_Syntax.Â Mine is -
```
@ 10800 IN TXT "v=spf1 +a:za3k.com +mx:za3k.com ~all"
-
```
-
You should also be verifying this on your end as part of combating spam, but as far as outgoing mail all you need to do is add a TXT record to your DNS record.
Set your rDNSÂ (reverse DNS) if itâs not already. This should point at the same hostname reported by Postfix during SMTP. This will be handled by whoever assigns your IP address (in my case, my hosting provider).
Test your spam reputability usingÂ https://www.mail-tester.comÂ orÂ https://www.port25.com/support/authentication-center/email-verification. You can monitor if youâre on any blacklists atÂ http://mxtoolbox.com/blacklists.aspx.
Set up DMARC. DMARC declares your policy around DKIM being mandatory. You can set up whatever policy you like.Â Mine is -
```
_dmarc 10800 IN TXT "v=DMARC1;p=reject;aspf=s;adkim=s;pct=100;rua=mailto:postmaster@za3k.com"
```
-

My sources writing this:

https://wiki.archlinux.org/index.php/Postfix and http://www.postfix.org/postconf.5.html on Postfix
https://wiki.archlinux.org/index.php/DovecotÂ on Dovecot
https://help.ubuntu.com/community/PostfixVirtualMailBoxClamSmtpHowto on SASL auth integration between Dovecot and Postfix
https://www.digitalocean.com/community/tutorials/how-to-set-up-a-postfix-email-server-with-dovecot-dynamic-maildirs-and-lmtp and https://www.digitalocean.com/community/tutorials/how-to-configure-a-mail-server-using-postfix-dovecot-mysql-and-spamassassinÂ onÂ SQL integration in Dovecot
https://scaron.info/blog/debian-mail-spf-dkim.html for setting up SPF, DKIM, and rDNS

Takeaways

You can set up store-and-forward mail servers, so if your mail server goes down, you donât lose all the email for that period. Itâs generally a free thing.
Postfixâs configuration files were badly designed and crufty, so you might pick a different SMTP server.
Email was REALLY not designed to do authentication, which is why proving youâre not a spammer is so difficult. This would all be trivial with decent crypto baked in (or really, almost any backwards-incompatible change)
The option to specify a SQL query as a configuration file option is wonderful. Thanks, Dovecot.
Overall, although it was a lot of work, I do feel like it was worth it to run my own email server.

-
-
-eduardz says: -
-
- November 23, 2016 at 12:18 am
-
-
Hello,
-
Can you implement quota support from postgressql?
-
-
-Reply
-
-
- -
  -
  -admin says: -
  -
  - April 5, 2017 at 7:05 pm
  -
  -
  Not sure if this is âwill you doâ or âis it possibleâ. I will not do anything to set up quotaâI have one user (me) so it doesnât make sense. I encourage you to link if you do.
  -
  -
  -Reply
  -
  -
-
-
-
-Tudor says: -
-
- February 23, 2017 at 2:45 am
-
-
In the âuseradd -d -M -d /var/mail/vmail âshell=/usr/bin/nologin -u 5000 -g vmail vmailâ I think the first â-dâ shouldnât be there
-
Thanks for the post
-
-
-Reply
-
-
- -
  -
  -admin says: -
  -
  - April 5, 2017 at 7:06 pm
  -
  -
  Thanks for the corrections, glad you found it useful.
  -
  -
  -Reply
  -
  -
-
-
-
-Tudor says: -
-
- February 23, 2017 at 2:54 am
-
-
Also, for Postgresql the correct commands to create users are:
-CREATE USER postfix PASSWORD âXXXâ;
-CREATE USER dovecot PASSWORD âXXXâ;
-
-
-Reply
-
-
-
-
-Tudor says: -
-
- February 23, 2017 at 3:22 am
-
-
Also, there shouldnât be any comma after the VALUES (
- âza3kâ,
- âza3k.comâ,
- â{SHA512}â¦â¦â¦â¦â¦â¦â¦â¦â¦â¦â¦â¦â¦â¦â¦â¦â¦â¦â¦â¦.==â part
-
-
-Reply
-
-
-
-
-S says: -
-
- May 12, 2018 at 9:36 am
-
-
A little sloppy â you should have simply shown the entire contents of each file one by one, with descriptions in comments or whatever, instead of breaking them up into snippets â and there are a few errors and omissions (I canât recall which ones now)
-However, using this guide and some googling, I was able to get a mailserver working â which is more than I can say for any other guide; so, thanks.
-
-
-Reply
-
-
-
-
-Korilius says: -
-
- July 17, 2021 at 3:56 pm
-
-
A guide that could have been great but tripped at the finish line. A lot of errors in SQL and elsewhere along with step reorganization.
-
-
-Reply
-
-
- -
  -
  -admin says: -
  -
  - July 17, 2021 at 4:52 pm
  -
  -
  If you have any specific errors you can mention, Iâd be happy to fix stuff.
  -
  But yeah, thatâs the problem with writing a guide after you do something instead of during/before.
  -
  -
  -Reply
  -
  -
-

A friend of mine, Kragen Javier Sitaker has been designing something he calls the zorzpad (see link below). I can never remember the name, so as a joke my version became the âzorch padâ. We live on opposite sides of the globe, but weâve picked up the same or similar hardware, and have been having fun developing the hardware and software together.

The basic idea of the Zorchpad is to have one computer, indefinitely. It should keep working until you die. That means no battery that runs out, and no parts that go bad (and of course, no requirements to âphone homeâ for the latest update via wifi!). This is not your standard computer, and weâve been trying a lot of experimental things. One of the main requirements is that everything be very low-power. He picked out the excellent apollo3 processor, which theoretically runs at around 1mW. In general, the zorchpad is made of closed-source hardware.

Since Iâve realized this will be a long project, Iâm going to post it piece-by-piece as I make progress. Below is a demo of the display.

- -

The graphics demo shows, in order:

a title screen
a flashing screen (to show graphics-mode framerate)
a demo of font rendering. weâre using the fixed-width font tamsyn.
a munching squares animation
a demo of how fast âtext-modeâ updates would be

Weâre using a memory-in-pixel LCD. The only manufacturer is Sharp LCD. You have have seen these before in things like the Pebble watchâtheyâve very low-power except when youâre updating. This particular screen is quite tinyâ240x400px display (which is fine with me), but only 1.39Ã2.31 inches (35x59mm). The only bigger screen available in this technology is 67x89mm, a bit lower resolution, and out of stock. As soon as itâs in stock I plan to switch to it.

According to the datasheet, the screen consumes 0.05-0.25mW without an update, and perhaps 0.175-0.35mW updating once per second. We havenât yet measured the real power consumption for any of the components.

The most obvious alternative is e-ink. E-ink has a muuuch slower refresh rate (maybe 1Hz if you hack it), and uses no power when not updating. Unfortunately it uses orders of magnitude more power for an update. Also, you can get much larger e-ink screens. The final zorchpad might have one, both or something else entirely! Weâre in an experimentation phase.

Datasheets, a bill of materials, and all source code can be found in my zorchpad repo. Also check out Kragenâs zorzpad repo.

-
-
-JenniferRM says: -
-
- April 20, 2024 at 9:52 am
-
-
Very cool idea. I was imagining form factors, and was thinking briefly about somehow having a sort of laptop arrangement where an e-ink screen is visible through glass when it is closed and in some kind of âprotective storage modeâ, and then visible directly from âthe proper side of the e-ink screenâ after it opens up. This is probably impossible using default hardware options, but searching around to confirm this lead to some interesting links and unusual design demos.
-
Hereâs ~12 year old thread from someone who wants a computer to use in an off grid cabin.
-https://forums.tomshardware.com/threads/ultra-low-power-pc-to-run-off-solar-power.1375469/
-
Here is a 2023 video about the revived product âGVUIDOâ (pronounced Guido) optimized for musicians where part of the UI involves covering a light sensor.
-https://www.youtube.com/watch?v=wTIf9wjm0y8
-
One thing that struck me is that a lot of people might want a Xorchpad to stick INSIDE a larger system (out in nature? as part of a science package?) that might install a rugged solar panel and battery. Then a small USB-C cord might provide a Xorchpad inside the unit with power, and want some intelligence to come out of the same cord, in exchange?
-
Iâm not sure if a USB-C is consistent with the vision, however. Maybe the âminimum powerâ for that is far above the âmaximim powerâ that would not blow the Sourcepadâs circuits?
-
-
-Reply
-
-
- -
  -
  -admin says: -
  -
  - April 20, 2024 at 10:24 am
  -
  -
  Try using numbers, instead of words! It works better for comparing power usage.
  -
  The âlow-power computerâ someone wants from 12 years ago should be 3W. The Zorchpad is designed to run at 0.001W.
  -
  Iâm not sure what GVUIDO has to do with anything, was there a reason you linked that?
  -
  The Zorchpad could easily be powered by any standard cable, yes. But if you need a larger system to power it, youâre missing the point.
  -
  âBlowing circuitsâ is not a correct intuition for designing low-power electronics. Go learn about Ohmâs Law!
  -
  -
  -Reply
  -
  -
-

I added an IRC server to my growing list of services. You can access it at:

irc.za3k.com
Port 6667

Or via the webchat, which I recommend.

Iâve also recently updated my home page to look much prettier, in imitation of a Computer Craft cheatsheet Iâve been working on.

Iâm one of the designers for the yearly April Fools Puzzle Contest on IRC.

Please feel free to join at https://blog.ircpuzzles.org/. The idea is that you solve puzzles in a chatroom, and get the password to the next chatroom, and so on. If youâre not familiar with IRC, donât worryâa link is provided to connect in your browser, too.

Itâs a lot of fun, and I hope you enjoy!

P.S. The contest should be up for a while, so donât worry about being late to the party!

I made a site for your constant refresh needs: http://isrickandmortyout.com/

I made a site for your constant refresh needs:Â http://isrickandmortyout.com/

Iâve been keeping a time log since somewhere around 2011. A time log is a journal with a complete record of everything I do. Iâve become very consistent about it, so this seemed like a good time to write up my current habits for anyone interested.

This is going to be a mixture of information about life-logging, how I organize things, and my current schedule, because theyâre not really separate things.

Thereâs an interesting story about how I systematically broke everyone one of my habits, and it took me 17 years to get in a daily routine after that, but thatâs a story for another time.

If youâre curious, Iâd guess it takes me 2 hours a week spread out to do my life-logging, 1 hour to type it up, and 1-2 hours to do my weekly review. In my mind the original life-logging doesnât cost me anything because itâs so automatic, itâs zero-energy, and it has some psychic benefits. By psychic benefits Iâm talking about the same kind of thing you get from GTDâyouâre not constantly thinking about or trying to remember things that are already written down in a trusted system. Typing it up and review are not free.

Time log (2011-)

I keep a written (pen and paper) time log which I normally just call my âlog bookâ. Each entry has the current time and what Iâm doing. I typically record an entry either when I start an activity, finish one, or notice Iâve switched activities. Iâm on volume 9.

Todayâs page starts like this (italics are censorship or words added for clarity):

Date: 2019-12-17, Tue
12:02pm	Woke up on my own slightly before alarm. Dream about [â¦]. (7h12m sleep)
	[100ml yellow rockstar recovery. (33mg caffein, 400mg taurineâfrom front material)]
	Morning data log (see below)
	Brushed teeth
12:55pm	Cancelled torrent verificationâI already know this will fail
	Responded to gnu coreutils âdateâ thread
	health stuff
2:02pm	Trying qutebrowser. Feels very productive.
2:04pm	[Coke Zero Vanilla, 1 can]

Iâm not fastidious about what the time represents. The questions I most often ask are âwhen did this happen roughlyâ and âdo I have any big portions of my day Iâm not time-loggingâ. Iâm less concerned with exactly how long I spent doing each particular activity.

There are some things I try to consistently write down every single time, including:

Exactly when I woke up, especially if I donât use the computer first thing (see âSleep Logâ below)
Any dream if I remember it
Any food or drink I consume, with enough information that I could generate nutritional facts if I wanted. I omit food amounts if itâs a pain to measure. 1 package of ramen: yes, 125g chicken curry: no. I put food and drinks in hard brackets: []
Watching a movie, TV show, youtube, or reading a book. I used to underline these, now Iâm trying putting them between underscores: _. Iâm switching to write these in a computer-understandable way but itâs a work in progress.
Anything health-related, including symptoms, drugs I took, and bathroom visits. Drugs are a type of food [], the rest is freeform.
Travel from point A to B
Phone calls. I donât always manage this one. While youâre picking up the phone is a really garbage time to try and write something.
Any time I change timezones
Any time I work on a project for more than a couple minutes
âWhere did that time goâ: one of the goals here is to have no huge gaps. If I spent time browsing the web or researching, some vague notes on what about. If I talk to someone in person, noting who and possibly what topics we talked about (talking in person often feels like minutes in my head but hours on the clock).

Here are things I donât write down:

Information that Iâve put elsewhere. See below for specifics on what else I have! This one isnât hard and fast, but Iâm a believer in things being in âexactly one placeâ as much as possibleâI do make some exceptions since Iâm working with paper
General-purpose notetaking, thoughts about whatâs going on, TODO lists, etc. This is just a boring olâ record of time. I do sometimes jot down TO-DOs when out of the house since this is the only paper I carry on me, but at the rate of 1-3 a week. I also may write down where Iâm at in a really long-running computer project, just to make sure I can find it later.
Anything a human shouldnât have to write or read. For example, I could write down the youtube URL or the UPC code of everything I buyâ¦ but nobody has time for that, and Iâd only write it down wrong.

At the front of the book I have a table with guides to abbreviations, ingredients in things I have often (ex. caffein amounts or recipes). In the back is my bookkeeping section (see below).

I am currently using the Leuchtturm1917 gridded notebook, with date labels at the top of the page. Iâve been experimenting with felt micron pensâIâm looking for something that can write easily, but wonât smear when I close the book. Iâve used Moleskins in the pastâI stopped using them because 2 of 5 split at the spine for me. Leuctturm seems a bit better but more expensiveâtime will tell.

One a week, I type up my time log up to the last page. Iâm working on my backlog slowly. This lets me search more easily. I have plans to someday cross-reference better in a computer system (for example, include nutritional info, link to youtube videos, etc).

Bookkeeping (2019-)

Fun fact: b-oo-kk-ee-ping is the only word in the English language with three consecutive double letters. Bookkeeping is keeping a record of what you earn and spend, or what you buy and sell.

For the most part, I pay for everything using a credit or debit card, which Iâve been doing since 16 so that I have a financial record for my own benefit. Most banks offer an easy export. I get paper copies, then once I download the PDFs from my bank, throw out the originals (Iâve checked one or two match the PDFs by hand). I use mint.com for the purpose of having a CSV export from my bak statements. I used to put this export online (currently broken, check back soon).

Starting a few months ago, I started keeping a weekly record by hand. Every time I spend money, Iâll put a $ symbol in my time log,

2:21am Amazon $

and add a bookkeeping entry (real thing is prettier).

2019-12-15, Sun
[ ] Amazon -29.21 -236.07
Choline citrate, 500g

The entry includes:

The date (2019-12-15)
Where I spent the money (Amazon)
How much money (29.21)
How much total Iâve spent this week (236.07)
What I bought (Choline citrate, 500g). If it was more than one thing, how much each item cost. Iâll try and write price-per-pound if Iâm buying bulk food or meat. If Iâm buying more than one of something, Iâll write how many I bought and how much each is. Iâd like to consistently write down how much of something I got (ex. 16oz of cheese) but I donât at all yet.
If itâs something that needs to be delivered, Iâll write a checkbox. Then when it arrives, Iâll check the box and write down the date it arrived to the right. This way I can easily scan and see if something never got delivered.

Since I use the same book for my time log and my bookkeeping, bookkeeping goes from right to left, two pages per week. At the end of the book, I keep

a running record of any debts I owe
any undelivered packages from the previous log book

During my weekly review process, I copy this information to my (digital) weekly review and add it up by category to check against my budget. I used to check it against my bank statements, but it takes forever and itâs easier to just be really good about writing down everything to start with. Checking totals and category totals is pretty time consuming the way I do it, Iâll probably automate it soon.

Budget

My current categories are:

taxes, bills, rent: Predictable expenses, no need to check these on a regular basis. I separate out medical bills in my summary, which are not regular.
travel, hard drives, moving: Big but one-off expenses. Currently I donât have a way to budget these.
charity: I aim for 10% of my income after taxes (a tithe)
other: The main budget category, I try to keep this at $1000/month ($240/week). I actually break it down into categories like âfoodâ, âgroceriesâ, and âluxuriesâ so I know what happened, as well as pulling out any single big expenses.

Weekly Schedule (2019-)

My current schedule is weekly:

Monday: Do meal planning for the week, and grocery shopping for the week if needed.
Tuesday: Cook food for the week.
Thursday: Batch day.
Do all the small chores (<1 hour) on one day. I aim for around 2-4 hours of chores, but Iâm fine skipping a batch day if I donât really have anything. I almost always clean my room and do laundry at minimum. I also have a running list of small tasks: call the doctor, clean the fridge, fix SSL certs.
Friday: Review day.
Iâll do a weekly review, and a monthly one if itâs the last weekly review of the month. Then Iâll type up the timelog up to that point in time. For my weekly review, which I do on my computer, I write down
- How much sleep I got on average
- What I did each day of the week (summary of that dayâs time log). Typically once I cut out really boring things (brush your teeth), food, movies, etc thereâs not all that much left.
- Accomplishments. Anything I got done this week. Also, any big milestones reached (finished X) even if the last step wasnât that impressive.
- Reflection/things learned: Did anything major happen? Did I learn any new facts? This is my time to look at the big picture and thing about how my life is going lately and where Iâd like it to go. Also, if anything especially good/bad happened, I try to think about why and how to make things go well next time.
- Finances. I copy down my expenses for the week and total them by category.
Saturday: Nothing planned.
Sunday: Nothing planned.

I havenât done batch cooking in a while, but Iâm also trying to run out my food supplies because Iâm about to move, so weâll see if it sticks around.

Daily Log (2019-)

Every morning, I record:

The date and time Iâm recording
How much sleep I got (but not when I went to sleep or woke up)
What day it is in my schedule
The temperature of the room
My body temperature (am I running a fever?)
How much exercise I got yesterday, in minutes (and what type)
My weight

I donât think it matters that much how you do these measurements, but itâs important to be consistent (for example, weight with clothes on/off?)

If I have a specific habit Iâm trying to pick up (say, brushing my teeth twice a day or meditating) I might record that for a while too each day. I used to record a mission for the day, but I dropped the habit.

Automatic Logs

I put all my computer logs in a single combined format, and sync them to a single location, starting in 2019. The format is basically <date> [<log name>:<computer name>] <Log entry>. I donât have a great process to view logs yet.

Sleep Log (2019-) / Keystoke Activity Log (2013-)

I log which hours I was asleep. I live alone and tend to fall asleep first thing after closing my laptop in bed, or at least with a video playing in the background, which makes this relatively easy. I keep a computer log of whether Iâm using my keyboard (I almost never do anything with just the mouse) for each minute using a custom-built keylogger (it records activity but not passwords).

Then I run it through a custom script (included in link) which says which broad periods I was active. The biggest inactive period in a day is when I was asleep.

~ $ sleep?
Report period: 2019-12-17 00:00:00 â 2019-12-17 16:21:06 (16:21:06s)
Inactive: 2019-12-17 04:50:18 â 2019-12-17 12:02:58 ( 7:12:40s)

I was asleep from 4:50am to 12:02pm. I make sure to write down when I wake up into my time log in case I donât use the computer first thing. This has been much better at guessing when I fell asleep than anything else Iâve tried.

If you donât fall asleep at a computer, I have some ideas around using a motion sensor (cheap webcams can see in the dark)

Chromium History Log (2013-)

I use Chromium as my only web browser. I export the history and bookmarks every time I do a backup, and put it all in a standard log format (basically time + URL). Currently I only record each history entry once.

For futureproofing, I archive every webpage I go to on an irregular basis (about once a year). Archiving pages doesnât work super well but itâs better than nothing.

Video/TV Log (2019-)

I watch my movies using noice, either directly on my television, or streamed from my media server to my laptop. When I start watching something, it automatically gets logged (including what the movie is, the path, how long it is etc). Same for when I stop, so I know if I quit early.

Youtube is included in my chromium history (see above). Sadly Iâm not sure I can get âhow much of this video did I watchâ from my formatâonly that I visited the video.

For futureproofing, I automatically archive every youtube video I watch.

Bash History (2011-)

This one is pretty simple. My Linux shell history (everything I run from the command line, which is basically everything I do outside a browser) is saved, forever. This one goes back to 2011 for my laptops.

Scanning (2014-)

I scan all documents I write, mail I get, etc. and generally throw out the originals. I organize everything by hand, and keep everything as image files.

I use a flat folder structure, which is to say I have a âscansâ folder and then a bunch of folders in it like âtaxes â 2019â. No nesting. This was my main takeaway from GTD for Hackers and I use flat folders for most digital organization.

I use the Doxie Go feed-through scanner (doesnât need a computer, writes directly to SD which I love). I recently got a Canon Lide 400 flatbed scanner (works on linux) which I use to scan bound books like my time log.

Who else does this stuff?

As far as I know I came up with this stuff independently. Iâve read plenty of time-management resources (which tend to be good) and experimental journaling resources (which tend to beâ¦ scarce?).

Lion Kimbro: âMake a complete map of every thought you thinkâ. General journaling. Inteview.
Fenn Lipowitz (my roommate): Time log, with an emphasis on being completely machine-readable. Being machine-readable means click for pretty graphs. I took inspiration from how machine-parsable this was recently, but I want to keep my freehand sections too.
Bryan Bishop (acquaintance): meetlog, a system for recording conversations and topics of conversation. Overall I didnât find this useful because I donât know hundreds of people. The format is so-so, largely because the author can type very fast, including real-time transcripts. I got the inspiration to write topics of conversation while talking from this. I do something similar if I spend a long time thinking or researching, too.
Bullet Journaling: I dunno, if youâre super lost and donât know how to write a journal/TODO list, some guy figured it out for you! Itâs just the basics that youâd figure out on your own, but it may save time. The site is better than the book. I independently invented most of their notation for TODO lists, I donât find it too useful for a journal. Other peoplesâ bullet journal pages are also useful, not just the original authorâs.

So have you ever used a web printer and it was great?

â¦

Yeah, me neither. Itâs probably possible on windows, but try to add more than one OS to the network and itâs horrible. And actually printing is a major pain in Linux anyway. Theoretically âlpâ and the like have no problem with remote printers, but I wanted something I understood. So today Iâm going to post my setup I use instead.

I have a computer physically connected to the printer. Letâs call it âprintserverâ. On that server there is a folder, /printme, which is constantly monitored by inode. Any file added to that directory is printed.

Suppose I downloaded cutecats.pdf and I want to print it. Then I run:

scp cutecats.pdf printserver:/printme
-

And voila, the cute cats get printed.

Hereâs the setup for the server:

Get the printer to work. This is the hard step.
Make a directory /printme. Add any missing users, add a new group called âprintâ and add everyone who needs to print to that, etc.

Set up /printme to be a tmpfs with the sticky bit set. (So we donât fill up the hard drive) -

/etc/fstab
-tmpfs           /printme        tmpfs   rw,nodev,nosuid,noexec,uid=nobody,gid=print,mode=1770,size=1G  0       0

Install incron and add this to the incrontab (of user âprintâ or âsudoâ): -
```
# incrontab -l
-/printme IN_CLOSE_WRITE,IN_MOVED_TO lp $@/$#
```
-
Note that this will preserve files after theyâre printed, because my server is low-volume enough I donât need to care.

This expands on my previous post about how to set up an email server.

Weâre going to set up a few spam filters in Dovecot under Debian. Weâre going to use Sieve, which lets the user set up whichever filters they want. However, weâre going to run a couple pre-bakedÂ spam filters regardless of what the user sets up.

Install Sieve. -

sudo apt-get install dovecot-sieveÂ dovecot-managesieved

Add Sieve toÂ Dovecot -

# /etc/dovecot/dovecot.conf
-# Sieve and ManageSieve
-protocols = $protocols sieve
-protocol lmtp {
- mail_plugins = $mail_plugins sieve
-}
-service managesieve-login {
- inet_listener sieve {
- port = 4190
- }
-}
-protocol sieve {
- managesieve_logout_format = bytes ( in=%i : out=%o )
-}
-plugin {
- # Settings for the Sieve and ManageSieve plugin
- sieve = file:~/sieve;active=~/.dovecot.sieve
- sieve_before = /etc/dovecot/sieve.d/
- sieve_dir = ~/sieve # For old version of ManageSieve
- #sieve_extensions = +vnd.dovecot.filter
- #sieve_plugins = sieve_extprograms
-}
-

Install and updateÂ SpamAssassin, a heuristic perl script for spam filtering. -

sudo apt-get install spamasssassin
-sudo sa-update

# /etc/default/spamassassin
-ENABLED=1
-#CRON=1 # Update automatically

# /etc/spamassassin/local.cf
-report_safe 0 # Don't modify headers
-

sudo service spamassassin start
-

Thereâs a lot of custom configuration and training you should do to get SpamAssassin to accurately categorize what you consider spam. Iâm including a minimal amount here. The following will train SpamAssassin system-wide based on what users sort into spam folders. -

#!/bin/sh
-# /etc/cron.daily/spamassassin-train
-all_folders() {
-        find /var/mail/vmail -type d -regextype posix-extended -regex '.*/cur|new$'
-}
-
-all_folders | grep "Spam" | sa-learn --spam -f - >/dev/null 2>/dev/null
-all_folders | grep -v "Spam" | sa-learn --ham -f - >/dev/null 2>/dev/null
-

Make Postfix run SpamAssassin as a filter, so that it can add headers as mail comes in. -

# /etc/postfix/master.cf
-smtp inet n - - - - smtpd
- -o content_filter=spamassassin
-# ...
-spamassassin unix - n n - - pipe user=debian-spamd argv=/usr/bin/spamc -f -e /usr/sbin/sendmail -oi -f ${sender} ${recipient}
-

sudo service postfix restart

Add SpamAssassin to Sieve. Dovecot (via Sieve) will now move messages with spam headers from SpamAssassin to your spam folder. Make sure you have a âSpamâ folder and that itâs set to autosubscribe. -

# /etc/dovecot/sieve.d/spam-assassin.sieve
-require ["fileinto"];
-# Move spam to spam folder
-if header :contains "X-Spam-Flag" "YES" {
- fileinto "Spam";
- # Stop here - if there are other rules, ignore them for spam messages
- stop;
-}

cd /etc/dovecot/sieve.d
-sudo sievec spam-assassin.sieve

Restart Dovecot -
```
sudo service dovecot restart
```
-
Test spam. TheÂ GTUBEÂ is designed toÂ definitely get rejected. Set the content of your email to this: -
```
XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X
```
-
You should also be able to create user-defined filters in Sieve, via the ManageSieve protocol. I tested this using a Sieve thunderbird extension. Youâre on your own here.

If you want a really good source of random numbers, you should get a hardware generator. But thereâs not a lot of great options out there, and most people looking into this get (understandably) paranoid about backdoors. But, thereâs a nice trick: if you combine multiple random sources together with xor, it doesnât matter if one is backdoored, as long as they arenât all backdoored. There are some exceptionsâif the backdoor is actively looking at the output, it can still break your system. But as long as youâre just generating some random pads, instead of making a kernel entropy pool, youâre fine with this trick.

So! We just need a bunch of sources of randomness. Hereâs the options Iâve tried:

/dev/urandom (40,000KB/s) â this is nearly a pseudo-random number generator, so itâs not that good. But itâs good to throw in just in case. [Learn about /dev/random vs /dev/urandom if you havenât. Then unlearn it again.]
random-stream (1,000 KB/s), an implementation of the merenne twister pseudo-random-number generator. A worse version of /dev/urandom, use that unless you donât trust the Linux kernel for some reason.
infnoise (20-23 KB/s), a USB hardware random number generator. Optionally whitens using keccak. Mine is unfortunately broken (probably?) and outputs âUSB read errorâ after a while
OneRNG (55 KiB/s), a USB hardware random number generator. I use a custom script which outputs raw data instead of the provided scripts (although they look totally innocuous, do recommend
/dev/hwrng (123 KB/s), which accesses the hardware random number generator built into the raspberry pi. this device is provided by the raspbian package rng-tools. I learned about this option here
rdrand-gen (5,800 KB/s), a command-line tool to output random numbers from the Intel hardware generator instruction, RDRAND.

At the end, you can use my xor program to combine the streams/files. Make sure to use limit the output size if using filesâby default it does not stop outputting data until EVERY file ends. The speed of the combined stream is at most going to be the slowest component (plus a little slowdown to xor everything). Hereâs my final command line:

#!/bin/bash
-# Fill up the folder with 1 GB one-time pads. Requires 'rng-tools' and a raspberry pi. Run as sudo to access /dev/hwrng.
-while true; do
-  sh onerng.sh | dd bs=1K count=1000000 of=tmp-onerng.pad 2>/dev/null
-  infnoise --raw | dd bs=1K count=1000000 of=tmp-infnoise.pad 2>/dev/null
-  xor tmp-onerng.pad tmp-infnoise.pad /dev/urandom /dev/hwrng | dd bs=1K count=1000000 of=/home/pi/pads/1GB-`\date +%Y-%m-%d-%H%M%S`.pad 2>/dev/null;
-done

Great, now you have a good one-time-pad and can join ok-mixnet ð

P.S. If you really know what youâre doing and like shooting yourself in the foot, you could try combining and whitening entropy sources with a randomness sponge like keccak instead.

-
-
-jiacheng hao says: -
-
- March 3, 2024 at 9:04 pm
-
-
Hello, I think you are truely right about the TRNG. I am a researcher who specializes in designing TRNG. And now I have a TRNG chip with PCB support USB2.0. And the speed can be up to 30Mbps. It can pass NIST 800-22 and 800-90B. Are you interested in that? Looking forward to your reply!!
-
-
-Reply
-
-
-
-
-admin says: -
-
- March 3, 2024 at 11:36 pm
-
-
Interested in what way?
-
Is your TRNG open-source?
-
Where do you research?
-
-
-Reply
-
-

I decided to post all of my purchases/income. This isnât something I was totally comfortable with, but I couldnât think of good reasons not to, and my default position is to release information. I think this is especially interesting since itâs not something Iâve seen made available before. Link: http://za3k.com/money.html

I think the analysis may be useful to other hackers, as people tend to be insane and cost-insensitive around money. I think having another personsâs finances to look at for comparison is something Iâve wanted for various reasons at various times, and itâs not commonly available. My selfish motivations are to get other people to tell me how I should be saving lots of money, and to feel like my financial decisions are under scrutiny (which is good and bad).

-
-
-anon says: -
-
- February 7, 2019 at 6:19 am
-
-
is offline
-
-
-Reply
-
-
- -
  -
  -admin says: -
  -
  - December 17, 2019 at 7:41 pm
  -
  -
  Yeah, my export broke a while back. This should be fixed indefinitely, but covers 2011-2016 only now.
  -
  -
  -Reply
  -
  -
-

I recently made an art project as a birthday gift for a young friend of mine.

I hadnât seen the exact technique I invented to make stencils posted anywhere, so I figured Iâd share it. I donât think this is a good method, but maybe it will inspire someone to do something better.

I started with a blank tile. On top, I put down a layer of painterâs tape (basically masking tape, but a little less stretchy).

I printed and taped a piece of paper on top (made using this online tool with the font BreeSerif):

I used an x-acto knife to cut through both the top paper and paper, then removed the paper. Then I peeled the letters out of the painterâs tape.

I painted the letters with acrylic, let them dry, and removed the tape. In retrospect, it probably would have been easier to remove the tape wet, because acrylic paint is a little stretchy and I went over the lines.

The letters happily lifted right off the glazed tile, which hadnât been sanded or anything. I added a heavy coat of modge-podge spray sealant, which kept everything in place after drying.

Finally, I used a masonry bit to drill screw holes in the tile, so it could be attached to a door.

She seemed to like it :). But now she wants to make one too. Iâll have to see if I can invent an easier way.

Today I added support for development of za3k.com using git:

# !/bin/sh
-# /git/bare-repos/za3k.com/hooks/post-update
-cd ~za3k/public_html
-env -i git pull
-echo "Deployed za3k.com"
-

and markdown support, via a cgi markdown wrapperÂ someone wrote for apache (yes, Iâm still using Apache).

Edit: I ended up wanting support for tables in markdown, so I used RubyâsÂ redcarpet markdown gem (the same thing Github uses, supports this style of tables as well as code blocks).

CGI support via http://blog.tonns.org/2013/10/enabling-markdown-on-your-apache.html

I play D&D. There are a thousand initiative trackers out there. Hereâs one I invented recently.

First, each player picks a Meeple to be their characterâs mini.

Four meeples on an index card, representing a wagon. — Four PCs on a wagon move over swampy terrain.

Quick, roll initiative! The players all roll, and so do the enemies. We grab a second meeple for each player, as well as second token for each enemy. This becomes the initiative tracker.

This is the initiative order. Itâs currently the red meeple heroâs turn. Next up will be the blue meeple hero, then the black cube enemy, and so on.

In this article, I migrate an existing debian 10 buster release, from an unencrypted root drive, to an encrypted root. I used a second hard drive because itâs saferâthis is NOT an in-place migration guide. We will be encrypting / (root) only, not /boot. My computer uses UEFI. This guide is specific to debianâI happen to know these steps would be different on Arch Linux, for example. They probably work great on a different debian version, and might even work on something debian-based like Ubuntu.

In part 2, I add an optional extra where root decrypts using a special USB stick rather than a keyboard passphrase, for unattended boot.

Apologies if I forget any stepsâI wrote this after I did the migration, and not during, so itâs not copy-paste.

Q: Why arenât we encrypting /boot too?

Encrypting /boot doesnât add much security. Anyone can guess whatâs on my /bootâitâs the same as on everyone debian distro. And encrypting /boot doesnât prevent tamperingâsomeone can easily replace my encrypted partition by an unencrypted one without my noticing. Something like Secure Boot would resist tampering, but still doesnât require an encrypted /boot.
I pull a special trick in part 2. Grub2âs has new built-in encryption support, which is what would allow encrypting /boot. But grub2 canât handle keyfiles or keyscripts as of writing, which I use.

How boot works

For anyone that doesnât know, hereâs how a typical boot process works:

Your computer has built-in firmware, which on my computer meets a standard called UEFI. On older computers this is called BIOS. The firmware is built-in, closed-source, and often specific to your computer. You can replace it with something open-source if you wish.
The firmware has some settings for what order to boot hard disks, CD drives, and USB sticks in. The firmware tries each option in turn, failing and using the next if needed.
At the beginning of each hard disk is a partition table, a VERY short info section containing information about what partitions are on the disk, and where they are. There are two partition table types: MBR (older) and GPT (newer). UEFI can only read GPT partition tables. The first thing the firmware does for each boot disk is read the partition table, to figure out which partitions are there.
For UEFI, the firmware looks for an âEFIâ partition on the boot diskâa special partition which contains bootloader executables. EFI always has a FAT filesystem on it. The firmware runs an EFI executable from the partitionâwhich one is configured in the UEFI settings. In my setup thereâs only one executableâthe grub2 bootloaderâso it runs that without special configuration.
Grub2 starts. The first thing Grub2 does isâ¦ read the partition table(s) again. It finds the /boot partition, which contains grub.cfg, and reads grub.cfg. (There is a file in the efi partition right next to the executable, which tells grub where and how to find /boot/grub.cfg. This second file is confusingly also called grub.cfg, so letâs forget it exists, we donât care about it).
Grub2 invokes the Linux Kernel specified in grub.cfg, with the options specified in grub.cfg, including the an option to use a particular initramfs. Both the Linux kernel and the initramfs are also in /boot.
Now the kernel starts, using the initramfs. initramfs is a tiny, compressed, read-only filesystem only used in the bootloading process. The initramfsâs only job is to find the real root filesystem and open it. grub2 is pretty smart/big, which means initramfs may not have anything left to do on your system before you added encryption. If youâre doing decryption, it happens here. This is also how Linux handles weird filesystems (ZFS, btrfs, squashfs), some network filesystems, or hardware the bootloader doesnât know about. At the end of the process, we now have switched over to the REAL root filesystem.
The kernel starts. We are now big boys who can take care of ourselves, and the bootloading process is over. The kernel always runs /bin/init from the filesystem, which on my system is a symlink to systemd. This does all the usual startup stuff (start any SSH server, print a bunch of messages to the console, show a graphical login, etc).

Setting up the encrypted disk

First off, I used TWO hard drivesâthis is not an in-place migration, and that way nothing is broken if you mess up. One disk was in my root, and stayed there the whole time. The other I connected via USB.

Hereâs the output of gdisk -l on my original disk:

Number  Start (sector)    End (sector)  Size       Code  Name
-   1            2048         1050623   512.0 MiB   EF00  # EFI, mounted at /boot/efi
-   2         1050624       354803711   168.7 GiB   8300  # ext4, mounted at /
-   3       354803712       488396799   63.7 GiB    8200  # swap

Here will be the final output of gdisk -l on the new disk:

Number  Start (sector)    End (sector)  Size       Code  Name
-   1            2048          526335   256.0 MiB   EF00  efi # EFI, mounted at /boot/efi
-   2         1050624       135268351   64.0 GiB    8200  swap # swap
-   3       135268352       937703054   382.6 GiB   8300  root_cipher # ext4-on-LUKS. ext4 mounted at /
-   4          526336         1050623   256.0 MiB   8300  boot # ext4, mounted at /boot

Stop anything else running. Weâre going to do a âliveâ copy from the running system, so at least stop doing anything else. Also most of the commands in this guide need root (sudo).
Format the new disk. I used gdisk and you must select a gpt partition table. Basically I just made everything match the original. The one change I need is to add a /boot partition, so grub2 will be able to do the second stage. I also added partition labels with the c gdisk command to all partitions: boot, root_cipher, efi, and swap. I decided Iâd like to be able to migrate to a larger disk later without updating a bunch of GUIDs, and filesystem or partition labels are a good method.
Add encryption. I like filesystem-on-LUKS, but most other debian guides use filesystem-in-LVM-on-LUKS. Youâll enter your new disk password twiceâonce to make an encrypted partition, once to open the partition.
cryptsetup luksFormat /dev/disk/by-partlabel/root_cipher
cryptsetup open /dev/disk-by-partlabel/root_cipher root
Make the filesystems. For my setup:
mkfs.ext4 /dev/disk/by-partlabel/root
mkfs.ext4 /dev/disk/by-partlabel/boot
mkfs.vfat /dev/disk/by-partlabel/efi
Mount all the new filesystems at /mnt. Make sure everything (cryptsetup included) uses EXACTLY the same mount paths (ex /dev/disk/by-partlabel/boot instead of /dev/sda1) as your final system will, because debian will examine your mounts to generate boot config files.
mount /dev/disk/by-partlabel/root /mnt
mkdir /mnt/boot && mount /dev/disk/by-partlabel/boot /mnt/boot
mkdir /mnt/boot/efi && mount /dev/disk/by-partlabel/efi /mnt/boot/efi
mkdir /mnt/dev && mount --bind /dev /mnt/dev # for chroot
mkdir /mnt/sys && mount --bind /sys /mnt/sys
mkdir /mnt/proc && mount --bind /dev /mnt/proc
Copy everything over. I used rsync -axAX, but you can also use cp -ax. To learn what all these options are, read the man page. Make sure to keep the trailing slashes in the folder paths for rsync.
rsync -xavHAX / /mnt/ --no-i-r --info=progress2
rsync -xavHAX /boot/ /mnt/boot/
rsync -xavHAX /boot/efi/ /mnt/boot/efi/
Chroot in. You will now be âinâ the new system using your existing kernel.
chroot /mnt
Edit /etc/crypttab. Add:
root PARTLABEL=root_cipher none luks
Edit /etc/fstab. Mine looks like this:
/dev/mapper/root / ext4 errors=remount-ro 0 1 PARTLABEL=boot /boot ext4 defaults,nofail 0 1 PARTLABEL=efi /boot/efi vfat umask=0077,nofail PARTLABEL=swap none swap sw,nofail 0 0 tmpfs /tmp tmpfs mode=1777,nosuid,nodev 0 0
Edit /etc/default/grub. On debian you donât need to edit GRUB_CMDLINE_LINUX.
GRUB_DISABLE_LINUX_UUID=true GRUB_ENABLE_LINUX_PARTLABEL=true
Run grub-install. This will install the bootloader to efi. I forget the options to run it withâ¦ sorry!
Run update-grub (with no options). This will update /boot/grub.cfg so it knows how to find your new drive. You can verify the file by hand if you know how.
Run update-initramfs (with no options). This will update the initramfs so it can decrypt your root drive.
If there were any warnings or errors printed in the last three steps, something is wrong. Figure out whatâit wonât boot otherwise. Especially make sure your /etc/fstab and /etc/crypttab exactly match what youâve already used to mount filesystems.
Exit the chroot. Make sure any changes are synced to disk (you can unmount everything under /mnt in reverse order to make sure if you want)
Shut down your computer. Remove your root disk and boot from the new one. It should work now, asking for your password during boot.
Once you boot successfully and verify everything mounted, you can remove the nofail from /etc/fstab if you want.
(In my case, I also set up the swap partition after successful boot.) Edit: Oh, also donât use unencrypted swap with encrypted root. That was dumb.

I had previously hand-rolled a status monitor, status.za3k.com, which I am in the process of replacing (new version). I am replacing it with a linux monitoring daemon, mon, which I recommend. It is targeted at working system administrators. âmonâ adds many features over my own system, but still has a very bare-bones feeling.

The old service, âsimple-statusâ worked as follows:

You visited the URL. Then, the status page would (live) kick of about 30 parallel jobs, to check the status of 30 services
The list of services is one-per-file in a the services.d directory.
For each service, it ran a short script, with no command line arguments.
All output is displayed in a simple html table, with the name of the service, the status (with color coding), and a short output line.
The script could return with a success (0) or non-success status code. If it returned success, that status line would display in green for success. If it failed, the line would be highlighted red for failure.
Scripts can be anything, but I wrote several utility functions to be called from scripts. For example, âping?â checks whether a host is pingable.
Each script was wrapped in timeout. If the script took too long to run, it would be highlighted yellow.
The reason all scripts ran every time, is to prevent a failure mode where the information could ever be stale without me noticing.

Mon works as follows

The list of 30 services is defined in /etc/mon/con.cf.
For each service, it runs a single-line command (monitor) with arguments. The hostname(s) are added to the command line automatically.
All output can be displayed in a simple html table, with the name of the service, the status (with color coding), the time of last and next run, and a short output line. Or, I use âmonshowâ, which is similar but in a text format.
Monitors can be anything, but several useful ones are provided in /usr/lib/mon/mon.d (on debian). For example the monitor âpingâ checks whether a host is pingable.
The script could return with a success (0) or non-success status code. If it returned success, the status line would display in green for success (on the web interface), or red for failure.
All scripts run periodically. A script have many states, not just âsuccessâ or âfailureâ. For example âuntestedâ (not yet run) or âdependency failingâ (I assume, not yet seen).

As you can see, the two have a very similar approach to the component scripts, which is natural in the Linux world. Here is a comparison.

âsimple-statusâ does exactly one thing. âmonâ has many features, but does the minimum possible to provide each.
âsimple-statusâ is stateless. âmonâ has state.
âsimple-statusâ runs on demand. âmonâ is a daemon which runs monitors periodically.
Input is different. âsimple-statusâ is one script which takes a timeout. âmonâ listens for trap signals and talks to clients who want to know its state.
both can show an HTML status page that looks about the same, with some CGI parameters accepted.
âmonâ can also show a text status page.
both run monitors which return success based on status code, and provide extra information as standard output. âmonâ scripts are expected to be able to run on a list of hosts, rather than just one.
âmonâ has a config file. âsimple-statusâ has no options.
âsimple-statusâ is simple (27 lines). âmonâ has longer code (4922 lines)
âsimple-statusâ is written in bash, and does not expose this. âmonâ is written in perl, all the monitors are written in perl, and it allows inline perl in the config file
âsimple-statusâ limits the execution time of monitors. âmonâ does not.
âmonâ allows alerting, which call an arbitrary program to deliver the alert (email is common)
âmonâ supports traps, which are active alerts
âmonâ supports watchdog/heartbeat style alerts, where if a trap is not regularly received, it marks a service as failed.
âmonâ supports dependencies
âmonâ allows defining a service for several hosts at once

Overall I think that âmonâ is much more complex, but only to add features, and it doesnât have a lot of features I wouldnât use. It still is pretty simple with a simple interface. I recommend it as both good, and overall better than my system.

My only complaint is that itâs basically impossible to Google, which is why Iâm writing a recommendation for it here.

A postal money order

Allow me to introduce you all to the postal money order. For $1.50, you can get the equivalent of a cashierâs check from the post office. It can only be cashed by whoever you make it out to, and itâs basically accepted as cash by every corporation. You can also just give someone a blank one, although thatâs riskier to carry around for the obvious reasons.

I was tired of checks bouncing. I canât be bothered to make sure my account remains such-and-such, which means it happens sometimes, especially times like now when Iâm poor. So I asked my landlord if I could pay by money orderâheâd never heard of them before, but seemed okay with it when I explained (heâs a really good guy!).

I went down to the bank and got out $2750, and headed to the post office. I asked for 9 money orders, each for $303. The postal worker really only made a couple funny faces about me being weird, although my friend said she was prettyÂ loud about my walking out with that much cash-equivalent, it went pretty well. And I immediately endorsed all the money orders so now they can lie around the hose safely.

Also, they come with attachable receipts (shown in the picture) in case you lose the check andÂ need a replacement, so thatâs nice.

My newest site:Â http://moreorcs.com/

The site generates orc-themed emails for you, which you can get emailed at (completely insecurely, itâs just a web address at mailinatorÂ to see the content).Â Please check out mailinatorâsÂ site, itâs a really neat project.

Some samples:

the last small poop orc (thelastsmallpooporc@moreorcs.com)
poop gross green blood thirsty orc
49 cross-eyed slightly intimidating poop dumb orcs
the last slightly intimidating orc
quite a lot of slightly intimidating small orcs
73 slightly intimidating small pretty orcs
smelly orc
a few orcs
lots and lots and lots and lots and lots and lots of orcs

https://www.rpgsolo.com/Â has a table for resolving yes/no questions, in turn taken from FU RPG. Roll a die:

Yes, andâ¦
Yesâ¦
Yes, butâ¦
No, butâ¦
Noâ¦
No, andâ¦

Their example:

I listen at the door. Do I hear anything? (I determine the odds are 50/50 so I click the â50/50â button in the Get Answer section and I get back the following.)
-
Yes, andâ¦
-
(Now itâs up to me to determine what that means. Since it says âandâ that means I got some kind of bonus. So I am going to interpret that to mean that from the sounds I am hearing I have received some extra information. So I type or say to myself),
-
I hear one person in the room. (Now I ask my next question.) Is the door locked?
-
No, butâ¦
-
(The answer is no but itâs not a total loss. I interpret what that means then type the following),
-
The door seems weak enough that I can probably kick it open.

So weâve gotten a base system for telling stories. We then added the following:

Whenever you roll a result, roll TWICE. In one universe, you get one result. In the other universe, you get another result. (With accompanying description). If you have more than three universes lying around, discard down to three. All actions/questions are for a particular universe declared by the players.
(addendum) Actually, roll two dice: only split the universe if the second die comes up â1â or â2â. Otherwise, answer the question normally. This speeds things up a bit.

It was pretty fun in practice. I recommend using a text file over paper, since youâre going to do a lot of copy-paste. We had more fun with no GM than with a GM. No firm result yet on sandbox-worldbuilding vs players in scenarios; both seemed all right.

Iâve had a couple people ask how my TODO list works, so hereâs what Iâve been doing for the last few years. I have four lists in total: a calendar, a yearly list, a daily list, and a master list.

A calendar.

The calendar has anything that needs to be done on a specific day. Birthday reminders, doctorâs appointments, and weekly activities like board game night or trash day. Youâve seen calendars. This is nothing interesting.

A yearly goals list

A yearly list of my goals for the year. I typically have 5-15 goals, and finish half of them.

This is mostly for motivation and focus. I donât look at it much, and often only write it a third of the way into the year.

You can ignore this one.

Daily TODO list

A daily TODO list, written on paper. I throw it out at the end of each day, without copying anything off it. (I actually scan it, but I never look at the scans). This one I find very helpful.

Master TODO list

A âmasterâ TODO list, consisting of everything I want to get done long term. I store this as a text file.

Each task is a one-line description.

I sort tasks into four categories:

Tasks that will take under an hour
Tasks that will take under a day (but more than an hour)
Tasks that will take less than a week
Tasks that will take more than a week

At the very top is just a list of all my task numbers, so I can see how many I have in each category, and skip down to them.

Tasks are marked as

[ ] Unfinished
[x]Finished (think â)
[X]Cancelled (think â, decided not to do it)
[/] Partially done (for very big tasks)
[>]Transferred to another system (doesnât happen in the master TODO system, but sometimes I do this from my journal or a daily TODO list to indicate I wrote it down in the master TODO system)

In addition, I have a few special categories:

Urgent tasks. Sometimes Iâll have things that really need to get done soon (but not âtodayâ, or theyâd go on the daily list). Taxes often fit in here.
âStuckâ tasks. If I have no idea how to proceed with a task, it goes in a special category.
âDoneâ tasks. These are waiting to be archived (which is why everything you see is un-done)
âFor funâ tasks. I try to keep a tasks which are just for fun in their own little section. Things like âlearn to make ice creamâ!

I try to minimize subtasks, in general. If I have a big task (clean the house), Iâll try to list it as âclean the bedroomâ, etc as seperate tasks. If I have to, Iâll have a big task that references separate small tasks, but itâs the exception, and usually in the âmore than a weekâ category.

And thatâs about all I have to say.

Iâm experimenting with using Jekyll in place of wordpress. If you want you can check out [dead link] which containly my weekly review process.

If and when I do migrate, all the posts here will be magically migrated and the URLs will stay the same so links donât break.

Edit: I discontinued this experiment. Itâs too hard to migrate the old stuff and keep it looking good, and Iâd rather keep everything in one system.

I downloaded all 27 million Go games from online-go.com, aka OGS, with permission. They are available on Internet Archive or here as SGF files or JSON. You can use them for whatever you like.

I made a new cryptosystem called OK-Mixnet. It has âperfectâ security, as opposed to the usual pretty-good security. (Of course, itâs not magicâif your computer is hacked, the cryptosystem isnât gonna protect your data). Despite the name, itâs not really a mixnet per se, it just similarly defends against SIGINT.

A writeup is here: https://za3k.com/ok-mixnet.md

The alpha codebase is here: https://github.com/za3k/ok-mixnet

Let me know if youâd like to join the open alpha. Email me your username and IP (youâll need to forward a port).

Pakistan has blocked access to Wikipedia. Old Wikipedia is now available in urdu, and has the same content.

We are working on more clearly communicating the Old Wikipedia is not Wikipedia in Urdu like we do in Englishâtranslation help would be welcome!

https://ur.oldwikipedia.org

Prefer the old layout of Wikipedia? A couple friends and I made oldwikipedia.org

Hope you enjoy.

Iâve written a few simple storytelling games or RPGs lately. Theyâre all under two pages of rules, and currently theyâre all untested (first test is Monday!).

Ultimate Archwizards vs the Dark Lord Game for 3-6 players including one GM. Guessing 2-4 hours. Imagine the final episode of a fight animeâeveryone is mega-level powered. Relatively goofy. Suitable for beginner players, would work with a beginner GM too. Designed with zero prep in mind.
No, this cannot be! I AM INVINCIBLE! Game for 2-8 players, no GM. More fun with 4+. Guessing 30 minutes. Heroes want to kill Villains, Villains also want to be killed. Villains therefore send wave after slightly harder wave of enemies at the Heroes to level them up. Designed to be comfortable for complete beginners, while letting expert storytellers play in the same group. About half storytelling, half stats. Some gameplay is probably similar to Munchkin, but I havenât really played Munchkin. Thereâs a little prep at the start for Villains (5 minutes), almost none for Heroes. The only one of the three that needs playtesting to balance.
Ninjas Ninjas Ninjas! Game for 3 players exactly, no GM. Not for beginners. Guessing 5-30 minutes once you learn the rules (up to you). Frantically fast storytelling, challenges are 30-60 seconds each. There are several roles including the main ânarratorâ, which players swap often. The main goal is to show how cool your ninja team is, but you do also complete your mission. Can be played with nothing, not even paper. Could be expanded to work with 4 or 5 players, but would need more work.

Will post my two spy games in a bit once I type them up!

One of my archiving and backup contingencies is taking one screenshot per minute. You can also use this to get a good idea of how you spend your day, by turning it into a movie. Although with a tiling window manager like I use, itâs a headache to watch.

I send the screenshots over to another machine for storage, so theyâre not cluttering my laptop. It uses up 10-20GB per year.

Iâll go over my exact setup below in case anyone is interested in doing the same:

/bin/screenlog

GPG_KEY=Zachary
-TEMPLATE=/var/screenlog/%Y-%m-%d/%Y-%m-%d.%H:%M:%S.jpg
-export DISPLAY=:0
-export XAUTHORITY=/tmp/XAuthority
-
-IMG=$(\date +$TEMPLATE)
-mkdir -p $(dirname "$IMG")
-scrot "$IMG"
-gpg --encrypt -r "$GPG_KEY" "$IMG"
-shred -zu "$IMG"

The script

Prints everything to stderr if you run it manually
Makes a per-day directory. We store everything in /var/screenlog/2022-07-10/ for the day
Takes a screenshot. By default, crontab doesnât have X Windows (graphics) access. To allow it, the XAuthority file which allows access needs to be somewhere my crontab can reliably access. I picked /tmp/XAuthority. It doesnât need any unusual permissions, but the default location has some random characters in it.
GPG-encrypts the screenshot with a public key and deletes the original. This is extra protection in case my backups somehow get shared, so I donât literally leak all my habits, passwords, etc. I just use my standard key so I donât lose it. Itâs public-key crypto, so put the public key on your laptop. Put the private key on neither, one, or both, depending on which you want to be able to read the photos.

/etc/cron.d/screenlog

* * * * * zachary  /bin/screenlog
-20  * * * * zachary  rsync --remove-source-files -r /var/screenlog/ backup-machine:/data/screenlog/laptop
-30  * * * * zachary  rmdir /var/screenlog/*

Thatâs

Take a screenshot once every minute. Change the first * to */5 for every 5 minutes, and so on.
Copy over the gpg-encrypted screenshots hourly, deleting the local copy
Also hourly, delete empty per-day folders after the contents are copied, so they donât clutter things

~/.profile

export XAUTHORITY=/tmp/XAuthority

I mentioned /bin/screenlog needs to know where XAuthority is. In Arch Linux this is all I need to do.

Iâm opening the NNTP server at nttp.za3k.com (TLS or unencrypted) to the public. These are the newsgroups currently on it. It is virtually zero-traffic (no users, but also users post little).

If you donât have a news reader, Thunderbird can do the job, or take a look at the list here.

art.agency.applied
-art.autonomy
-art.programming
-news.announce.newusers
-news.discuss
-research.fai
-research.fai.decisiontheory
-research.math
-research.philosophy
-research.strategy
-research.xrisk
-talk.math
-talk.news
-talk.other
-talk.personal
-talk.philosophy
-talk.programming
-talk.psychology
-talk.reading
-talk.reading.collaborative
-talk.science
-talk.writing
-talk.writing.collaborative
-test.posts

I copied the list of songs I favoritedÂ fromÂ Pandora.

Previous work:

OptarÂ (OPTical ARchive), a 2-D barcode format (github)
Paperback

I wanted (for fun) to see if I could get data stored in paper formats. Iâd read the previous work, and people put a lot of thought into density, but not a lot of thought into ease of retreival. First off, acid-free paper lasts 500 years or so, which is plenty long enough compared to any environmental stresses (moisture, etc) I expect on any paper I have.

Optar gets a density of 200kB / A4 page. By default, it requires a 600dpi printer, and a 600+dpi scanner. It has 3-of-12 bit redundancy using Golay codes, and spaces out the bits in an okay fashion.

Paperback gets a (theoretical) density of 500kB / A4 page. It needs a 600dpi printer, and a ~900dpi scanner. Â It has configurable redundancy using Reed-Solomon codes. It looks completely unusable in practice (alignment issues, aside from being Windows-only).

Okay, so I think these are all stupid, because you need some custom software to decode them, which in any case where youâre decoding data stored on paper you probably donât have that. I want to use standard barcodes, even if theyâre going to be lower density. Letâs look at our options.Â Iâm going to skip linear barcodes (low-density) and color barcodes (printing in color is expensive). Â Since we need space between symbols, we want to pick the biggest versions of each code we can. For one, whitespace around codes is going to dominate actual codeÂ density for layout efficiency, and larger symbols are usually more dense. For another thing, we want to scan as few symbols as possible if weâre doing them one at a time.

AztecÂ From 15Ã15 to 151Ã151 square pixels.Â 1914 bytes maximum. Configurable Reed-Solomon error correction.

Density: 11.9 pixels per byte

Data Matrix From 10Ã10 to 144Ã144 square pixels.Â 1555 bytes maximum. Large, non-configurable error correction.

Density: 13.3 pixels per byte

QR Code From 21Ã21 to 177Ã177 square pixels.Â 2,953 bytes maximum.Â Somewhat configurable Reed-Solomon error correction.

Density: 10.6 pixels per byte

PDF417 17 heightÂ by 90-583Â width. Â 1100Â bytes maximum.Â Configurable Reed-Solomon error correction. PDF417 is a stacked linear barcode, and can be scanned by much simpler scanners instead of cameras. It also has built in cross-symbol linking (MacroPDF417), meaning you can scan a sequence of codes before getting outputâhandy for getting software to automatically link all the codesÂ on a page.

Density: 9.01 pixels per byte

QR codesÂ and PDF417 look like our contenders. PDF417 turns out to not scan well (at all, but especially at large symbol sizes), so despite some nice features letâs pick QR codes.Â Back when I worked on a digital libraryÂ I made a component to generate QR codes on the fly, and I know how to scan them on my phone and webcam already from that, so it would be pretty easy to use them.

What density can we get on a sheet of A4 paper (8.25 in Ã 11.00 in, or 7.75in x 10.50in with half-inch margins)? I trustÂ optarâs estimateÂ (600 dpi = 200 pixels per inch) for printed/scanned pages since they seemed to test things. A max-size QR code is 144Ã144 pixels, or 0.72 x 0.72 inches at maximum density. We can fit 10 x 14 = 140 QR codes with maximum density on the page, less if we want decent spacing. Thatâs 140 QR codes x (2,953 bytes per QR code) = 413420 bytes = 413K per pageÂ before error correction.

Thatâs totally comparable to the other approaches above, and you can read the results with off-the-shelf software. Â Bam.

A small font I designed. Iâm pretty proud of it.

I hacked up a little to-do app in an hour. The emphasis is on focus.

Itâs linux only. You can find it in my short-programs repo on github.

Between 2020-03-14 and 2020-12-03 I ran an experimental computer storage setup. I movied or copied 90% of my files into a content-addressable storage system. Iâm doing a writeup of why I did it, how I did it, and why I stopped. My hope is that it will be useful to anyone considering using a similar system.

The assumption behind this setup, is that 99% of my files never change, so itâs fine to store only one, static copy of them. (Think movies, photosâ¦ theyâre most of your computer space, and youâre never going to modify them). There are files you change, I just didnât put them into this system. If you run a database, this ainât for you.

Because I have quite a lot of files and 42 drives (7 in my computer, ~35 in a huge media server chassis), there is a problem of how to organize files across drives. To explain why itâs a problem, letâs look at the two default approaches:

One Block Device / RAID 0. Use some form of system that unifies block devices, such as RAID0 or a ZFSâs striped vdevs. Writing files is very easy, you see a single 3000GB drive.
- Many forms of RAID0 use striping. Striping splits each file across all available drives. 42 drives could spin up to read one file (wasteful).
- You need all the drives mounted to read anythingâI have ~40 drives, and Iâd like a solution that works if I move and canât keep my giant media server running. Also, itâs just more reassuring that nothing can fail if you can read each drive individually.
JBOD / Just a Bunch of Disks. Label each drive with a category (ex. âmoviesâ), and mount them individually.
- Itâs hard to aim for 100% (or even >80%) drive use. Say you have 4x 1000GB drives, and you have 800GB movies, 800GB home video footage, 100GB photographs, and 300GB datasets. How do you arrange that? One drive per dataset is pretty wasteful, as everything fits on 3. But, with three drives, youâll need to split at least one dataset across drives. Say you put together 800GB home video and 100GB photographs. If you get 200GB more photographs, do you split a 300 GB collection across drives, or move the entire thing to another drive? Itâs a lot of manual management and shifting things around for little reason.

Neither approach adds any redundancy, and 42 drives is a bit too many to deal with for most things. Step 1 is to split the 42 drives into 7 ZFS vdevs, each with 2-drive redundancy. That way, if a drive fails or there is a small data corruption (likely), everything will keep working. So now we only have to think about accessing 7 drives (but keep in mind, many physical drives will spin up for each disk access).

The ideal solution:

Will not involve a lot of manual management
Will fill up each drive in turn to 100%, rather than all drives at an equal %.
Will deduplicate identical content (this is a ânice to haveâ)
Will only involve accessing one drive to access one file
Will allow me to get and remove drives, ideally across heterogenous systems.

I decided a content-addressable system was ideal for me. That is, youâd look up each file by its hash. I donât like having an extra step to access files, so files would be accessed by symlinkâno frontend program. Also, it was important to me that I be able to transparently swap out the set of drives backing this. I wanted to make the content-addressable system basically a set of 7 content-addressable systems, and somehow wrap those all into one big content-addressable system with the same interface. Hereâs what I settled on:

(My drives are mounted as /zpool/bs0, /zpool/bs1, â¦ /zpool/bs6)
Files will be stored in each pool in turn by hash. So my movie âcat.mpgâ with sha hash â8323f58d8b92e6fdf190c56594ed767df90f1b6dâ gets stored in /zpool/bs0/83/23/f58d8b9 [shortened for readability]
Initially, we just copy files into the content-addressable system, we donât delete the original. Iâm cautious, and I wanted to make everything worked before getting rid of the originals.
To access a file, I used read-only unionfs-fuse for this. This checks each of /zpool/bs{0..6}/<hash> in turn. So in the final version, /data/movies/cat.mpg would be a symlink to â/bs-union/83/23/f58d8b9â
We store some extra metadata on the original file (if not replaced by a symlink) and the storied copyâwhat collection itâs part of, when it was added, how big it is, what itâs hash is, etc. I chose to use xattrs.

The plan here is that it would be really easy to swap out one backing blockstore of 30GB, for two of 20GBâjust copy the files to the new drives and add it to the unionfs.

Hereâs what went well:

No problems during developmentâonly copying files meant it was easy and safe to debug prototypes.
Everything was trivial to access (except see note about mounting disks below)
It was easy to add things to the system
Holding off on deleting the original content until I was 100% out of room on my room disks, meant it was easy to migrate off of, rather risk-free
Running the entire thing on top of zfs ZRAID2 was the right decision, I had no worries about failing drives or data corruption, despite a lot of hardware issues developing at one point.
My assumption that files would never change was correct. I made the unionfs filesystem read-only as a guard against error, but it was never a problem.
Migrating off the system went smoothly

Here are the implementation problems I found

I wrote the entire thing as bash scripts operating directly on files, which was OK for access and putting stuff in the store, but just awful for trying to get an overview of data or migrating things. I definitely should have used a database. I maybe should have used a programming language.
Because there was no database, there wasnât really any kind of regular check for orphans (content in the blobstore with no symlinks to it), and other similar checks.
unionfs-fuse suuucks. Every union filesystem Iâve tried sucks. Its read bandwidth is much lower than the component devices (unclear, probably), it doesnât cache where to look things up, and it has zero xattrs support (canât read xattrs from the underlying filesystem).
gotcha: zfs xattrs waste a lot of space by default, you need to reconfigure the default.

But the biggest problem was disk access patterns:

I thought I could cool 42 drives spinning, or at least a good portion of them. This was WRONG by far, and I am not sure how possible it is in a home setup. To give you an idea how bad this was, I had to write a monitor to shut off my computer if the drives went above 60C, and I was developing fevers in my bedroom (where the server is) from overheating. Not healthy.
unionfs has to check each backing drive. So we see 42 drives spin up. I have ideas on fixing this, but it doesnât deal with the other problems
To fix this, you could use double-indirection.
- Rather than pointing a symlink at a unionfs: /data/cat.mpg -> /bs-union/83/23/f58d8b9 (which accesses /zpool/bs0/83/23/f58d8b9)
- Point a symlink at another symlink that points directly to the data: /data/cat.mpg -> /bs-indirect/83/23/f58d8b9 -> /zpool/bs0/83/23/f58d8b9
The idea is that backing stores are kinda âwhatever, just shove it somewhereâ. But, actually it would be good to have a collection in one placeânot only to make it easy to copy, but to spin up only one drive when you go through everything in a collection. It might even be a good idea to have a separate drive for more frequently-accessed content. This wasnât a huge deal for me since migrating existing content meant it coincidentally ended up pretty localized.
Because I couldnât spin up all 42 drives, I had to keep a lot of the array unmounted, and mount the drives I needed into the unionfs manually.

So although I could have tried to fix things with double-indirection, I decided there were some other disadvantages to symlinks: estimating sizes, making offsite backups foolproof. I decided to migrate off the system entirely. The migration went well, although it required running all the drives at once, so some hardware errors popped up. Iâm currently on a semi-JBOD system (still on top of the same 7 ZRAID2 devices).

Hopefully this is useful to someone planning a similar system someday. If you learned something useful, or there are existing systems I should have used, feel free to leave a comment.

I made a minimal, printable TODO list. Enjoy!

Although the below directions work on Raspberry Pi, they should also work on any other system. The brother-provided driver does not run on arm processors[1] like the raspberry pi, so we will instead use the open-source brlaser[2].

Edit: This setup should also work on the following Brother monochrome printers, just substitute the name where needed:

brlaser 4, just install from package manager: DCP-1510, DCP-1600 series, DCP-7030, DCP-7040, DCP-7055, DCP-7055W, DCP-7060D, DCP-7065DN, DCP-7080, DCP-L2500D series, HL-1110 series, HL-1200 series, HL-L2300D series, HL-L2320D series, HL-L2340D series, HL-L2360D series, HL-L2375DW series, HL-L2390DW, MFC-1910W, MFC-7240, MFC-7360N, MFC-7365DN, MFC-7420, MFC-7460DN, MFC-7840W, MFC-L2710DW series
brlaser 6, follow full steps below: DCP-L2520D series, DCP-L2520DW series, DCP-L2540DW series (unclear, may only need 4), HL-2030 series, HL-2140 series, HL-2220 series, HL-2270DW series, HL-5030 series

Also, all these steps are command-line based, and you can do the whole setup headless (no monitor or keyboard) using SSH.

Get the latest raspbian image up and running on your pi, with working networking. At the time of writing the latest version is 10 (buster)âonce 11+ is released this will be much easier. I have written a convenience tool[3] for this step, but you can also find any number of standard guides. Log into your raspberry pi to run the following steps
(Option 1, not recommended) Upgrade to Debian 11 bullseye (current testing release). This is because we need brlaser 6, not brlaser 4 from debian 10 buster (current stable release). Then, install the print system and driver[2]:
sudo apt-get update && sudo apt-get install lpr cups ghostscript printer-driver-brlaser
(Option 2, recommended) Install âbrlaserâ from source.
1. Install print system and build tools
  sudo apt-get update && sudo apt-get install lpr cups ghostscript git cmake libcups2-dev libcupsimage2-dev
2. Download the source
  wget https://github.com/pdewacht/brlaser/archive/v6.tar.gz && tar xf v6.tar.gz
3. Build the source and install
  cd brlaser-6 && cmake . && make && sudo make install
Plug in the printer, verify that it shows up using sudo lsusb or sudo dmesg. (authorâs shameful note: if youâre not looking, I find it surprisingly easy to plug USB B into the ethernet jack)
Install the printer.
1. Run sudo lpinfo -v | grep usb to get the device name of your printer. It will be something like usb://Brother/HL-2270DW%20series?serial=D4N207646
  If youâre following this in the hopes that it will work on another printer, run sudo lpinfo -m | grep HL-2270DW to get the PPD file for your printer.
2. Install and enable the printer
  sudo lpadmin -p HL-2270DW -E -v usb://Brother/HL-2270DW%20series?serial=D4N207646 -m drv:///brlaser.drv/br2270dw.ppd
  Note, -p HL-2270DW is just the name Iâm using for the printer, feel free to name the printer whatever you like.
3. Enable the printer (did not work for me)
  sudo lpadmin -p HL-2270DW -E
4. (Optional) Set the printer as the default destination
  sudo lpoptions -d HL-2270DW
5. (Optional) Set any default options you want for the printer
  sudo lpoptions -p HL-2270DW -o media=letter
Test the printer (Iâm in the USA so we use âletterâ size paper, you can substitute whichever paper you have such as âa4â).
1. echo "Hello World" | PRINTER=HL-2270DW lp -o media=letter (Make sure anything prints)
2. cat <test document> | PRINTER=HL-2270DW lp -o media=letter (Print an actual test page to test alignment, etc)
3. cat <test document> | PRINTER=HL-2270DW lp -o media=letter -o sides=two-sided-short-edge (Make sure duplex works if you plan to use that)
(Optional) Set up an scp print server, so any file you copy to a /printme directory gets printed. For the 2270DW, I also have a /printme.duplex directory.

Links
[1] brother driver does not work on arm (also verified myself)
[2] brlaser, the open-source Brother printer driver
[3] rpi-setup, my convenience command-line script for headless raspberry pi setup
[4] stack overflow answer on how to install one package from testing in debian

-
-
-Joel says: -
-
- August 2, 2020 at 12:52 pm
-
-
There appears to be a typo in step 5A. The page currently reads âlpinfo -mâ but I believe should be âlpinfo -vâ. Per the man page, the m flag lists drivers and the v flag lists devices.
-
The USB device can be found in the -v output for step 5A but the driver can be found in the -m output for step 5B.
-
-
-Reply
-
-
-
-
-Scott says: -
-
- October 19, 2020 at 5:10 pm
-
-
Thank you so very much for this. Worked like a charm.
-
Any tips on how to print over the network?
-
-
-Reply
-
-
-
-
-rathesun01 says: -
-
- December 8, 2020 at 6:44 pm
-
-
Awesome post. Joel is supposedly correct. It should have been âlpinfo -vâ in the step 5A.
-
-
-Reply
-
-
-
-
-admin says: -
-
- December 12, 2020 at 1:31 pm
-
-
Corrected âlpinfo -vâ, thanks.
-No clue how to print over the network, sorry. Thatâs actually why I set up a raspberry pi to connect to the printer insteadâitâs my wifi interface.
-
-
-Reply
-
-
-
-
-steve says: -
-
- December 30, 2020 at 9:05 am
-
-
I have the Brother HL-2270DW, and I had to install from source to get it to work. Iâm not using USB, but port forwarding across via NAT to another internal network.
-
I just used the cups admin pages to complete setting things up. Before using this package, I picked some other closely related printer. It was printing the page, sucking it back in, then finally printing. Not terrible, but I was annoyed. Now itâs perfect!
-
For printing over the network, I used the socket::9100 setting. No idea the command line knobs or dials, just did it through the cups web page.
-
Another reason for me to do this is now my little rasperry pi zero w shows my printer as an AirPrint printer, so now I can print from my iDevices. Brother has their own app for this printer, but I was annoyed having to install a special app for this. My HL2270DW was made without AirPrint. Itâs a fine little machine and donât want to throw it out.
-
-
-Reply
-
-
- -
  -
  -Jaye Horn says: -
  -
  - February 18, 2021 at 3:19 pm
  -
  -
  How did you install from source? and how did you do the port forwarding across via NAT to another internal network? Iâm new to this so any help would be very much appreciated.
  -
  Thank you.
  -
  -
  -Reply
  -
  -
-
-
-
-Job says: -
-
- January 20, 2021 at 8:09 pm
-
-
Would this work for brother hl-l2395dw scanner?
-
-
-Reply
-
-
-
-
-Adam Trask says: -
-
- October 22, 2021 at 5:00 am
-
-
Thanks so much for posting this. This helped get my HL-L2300D working properly.
-
-
-Reply
-
-
-
-
-Danial Foster says: -
-
- December 24, 2021 at 1:39 am
-
-
>>> (authorâs shameful note: if youâre not looking, I find it
->>>surprisingly easy to plug USB B into the ethernet jack)
-
I smiled when I saw this because you are clearly a ding dong; that sounds like a rookie mistake.
-
I couldnât get your instructions to work, so I tried troubleshooting through Google. Couldnât figure it out. sudo lpinfo -v | grep usb wasnât showing jack.
-
It was because I had my USB B plugged into the ethernet jack of my BR-2270DW.
-
Everything works now. Thank you so much.
-
-
-Reply
-
-
-
-
-J Bot says: -
-
- January 3, 2023 at 4:03 pm
-
-
Any steps to do this with a wifi connected HL-2270DW? Thanks!
-
-
-Reply
-
-

One of the more useful things I did was to start logging all my technical problems. Whenever I hit a problem, I write an entry in problem-log.txt. Hereâs an example

2022-08-02
-Q: Why isn't the printer working? [ SOLVED ]
-A: sudo cupsenable HL-2270DW
-
-// This isn't in the problem log, but the issue is that CUPS will silently disable the printer if it thinks there's an issue. This can happen if you pull a USB cord mid-print.

I write the date, the question, and the answer. Later, when I have a tough or annoying problem, I try to grep problem-log.txt. Iâll add a note if I solve a problem using the log, too.

This was an interesting project to look at 5 years later. I didnât see benefits until 1-2 years later. It does not help me think through a problem. Itâs hard to remember to do. But, over time itâs built up and become invaluable to me. I hit a tricky problem, and I canât immediately find an answer on the web. I find out itâs in problem-log.txt. And, someoneâs written it exactly with my hardware (and sometimes even my folder names) correctly in there. Cool!

Hereâs another example:

2018-10-21
-Q: How do I connect to the small yellow router?

Not every problem gets solved. Oh well.

qr-backup is a program to back up digital documents to physical paper. Restore is done with a webcam, video camera, or scanner. Someday smart phone cameras will work.

Iâve been making some progress on qr-backup v1.1. So far Iâve added:

--restore, which does a one-step restore for you, instead of needing a bash one-line restore process
--encrypt provides password-based encryption
An automatic restore check that checks the generated PDF. This is mostly useful for me while maintaining qr-backup, but it also provides peace-of-mind to users.
--instructions to give more fine-tuned control over printing instructions. Thereâs a âplain englishâ explanation of how qr-backup works that you can attach to the backup.
--note for adding an arbitrary message to every sheet
Base-64 encoding is now per-QR code, each QR is self-contained.
Codes are labeled N01/50 instead of 01/50, to support more code types in the future.
Code cleanup of QR generation process.
Several bugfixes.

v1.1 will be released when I make qr-backup feature complete:

Erasure coding, so you only need 70% of the QRs to do a restore.
Improve webcam restore slightly.

v1.2 will focus on adding a GUI and support for Windows, Mac, and Android. Switching off zbar is a requirement to allow multi-platform support, and will likely improve storage density.

qr-backupÂ v1.1 is released. qr-backup is aÂ command-line Linux program. You can use it to back up a file as a series of QR codes. You can restore the QR codes using a webcam or scanner, and get back the original file.

The main features of qr-backup are ease-of-use and futureproofing (restore does not require qr-backup).

Please report any bugs on github. Once this is stable, I will do the first pip/package manager release. To test the alpha, check out the latest code using git.

See alsoÂ USAGEÂ andÂ extensiveÂ FAQ.

New features in v1.1:

Feature complete. New features are unlikely to be added. Future efforts will focus on quality, GUIs, and porting.
restore using qr-backup. Previously, the only restore was a bash one-liner (which still works).
- qr-backup --restoreÂ restores using the webcam
- qr-backup --restore IMAGE IMAGE IMAGEÂ restores from scanned images
After generating a PDF backup, qr-backup automatically does a digital test of the restore process
Erasure coding. Lose up to 30% of QRs and restore will still work, as long as you are using qr-backup to restore
Increased code density, which about cancels out the erasure coding.
Back up directories and files. qr-backup makes a .tar file for you
Option to use password protection (encryption)
Option to print multiple copies of every QR code
Option to randomize order of QR codes
Optionally print extra cover sheet instructions on how to restore. For long-term archivists.
Option to add custom notes and labels to each page
Improved support for using qr-backup in a pipe
Various bugfixes
SeeÂ CHANGELOGÂ for complete details

P.S. As a special request, if anyone is on OS X, let me know if it works for you?

I made a new project called qr-backup. Itâs a command-line program to back up any file to physical paper, using a number of QR codes. You can then restore it, even WITHOUT the qr-backup program, using the provided instructions.

Iâm fairly satisfied with its current state (can actually back up my files, makes a PDF). Thereâs definitely some future features Iâm looking forward to adding, though.

-
-
-scruss says: -
-
- June 9, 2021 at 6:58 am
-
-
nice! Iâve played with some similar ideas, using tar and QR Code output to a thermal printer. The used to be a thing (Twibright Optar, IIRC: itâs fallen off the web) that made full-page scannable codes that got an almost useful data density. But they werenât QR Codes, so needed their own decoder.
-
-
-Reply
-
-
- -
  -
  -admin says: -
  -
  - June 9, 2021 at 2:06 pm
  -
  -
  Actually, I link to it in the FAQ, itâs still on the web. An even better version was âPaperbackâ, but itâs 9 years unmaintainedâIâm looking into seeing if there is a maintained Linux port. Both do a lot of things right, even if they have a slightly different goal (high data density, over ease-of-use and foolproof restore).
  -
  -
  -Reply
  -
  -
- -
  -
  -admin says: -
  -
  - June 9, 2021 at 2:08 pm
  -
  -
  Also, feel free to recommend me a good, cheap thermal printer. I tried to do a âpoloroidâ thing (take a picture of yourself with webcam, immediately print to thermal) and found that mine was shit and the heat overexposed unrelated parts. QR codes seem like a reasonable application, although Iâd be concerned about the longevity of thermal paper for backups (can easily fade in heat).
  -
  Edit: If I remember correctly, I wanted to make a thermal-paper typewriter for a zine?
  -
  -
  -Reply
  -
  -
-

I was looking into building a raspberry pi based supercomputer lately. Hereâs the background research I did comparing pi models. Most of this information is sourced from raspberrypi.org. I was especially interested in which boot methods worked for which models, which is very scattered, as well as prices.

- -

My friend Callen tried to help me run a DC motor to roll my curtains up and down. We didnât make a ton of progress, but we had some fun making a little music.

Lately Iâve been messing about in Godot, a framework for making video games (similar to Unity).

I wanted to make a 3D game. In my game, you live in a geodesic dome, and canât go outside, because mumble mumble mumble poisonous atmosphere?.

A geodesic dome, I learned, is related to the icosahedron, or d20 from RPGs.

A simple dome is the top half of the icosahedron. As they get more complex, you divide each triangle into more and more smaller triangles.

Icosahedron getting more and more detailed. Geodesic domes are the top half of each sphere.

So to make a nice geodesic dome, we could find one (I failed), make one in Blender (too hard), or use some math to generate one in Godot. And to do that math, we need to know the list of 20 icosahedron faces. Which basically just needs the list of the 12 vertices!

Now, obviously you could look up the vertices, but I thought of a more fun way. Letâs put 12 points on a sphere, make them all repel each other (think magnetically, I guess), and see where on the sphere they slide to. Maybe they will all be spaced out evenly in the right places. Well, hereâs what it looks like:

- -

Soâ¦ kinda? It was certainly entertaining.

By the way, the correct coordinates for the vertices of an icosahedron inside a unit sphere are:

the top at (0, 1, 0)
the bottom at (0, -1, 0)
10 equally spaced points around a circle. they alternate going up and down below the center line.
(Â±1/â5,Â sin(angle), cos(angle)) [projected onto the sphere]

Hereâs how you make roasted chickpeas (aka garbanzo beans):

Set the oven to 400F.
Drain and emptyÂ a can or so of chickpeas into a collander and wash them
Dry the chickpeas (this is the hard step). I use paper towels, but I havenât figured out a way to not use a billion of them.
Put them in a short pan in the oven and cover them in olive oil. Toss them some with your hands to get them coated.
Cook for 20-30 minutes, shaking the pan so everything turns every 10 minutes. I like them crispy so I do 30 minutes.
Take them out and transfer them to a bowl. Add spices. I like salt, garlic powder, and pepper.

Iâve had a lot of trouble getting Minecraft Forge to run headless. They have a friendly installer option that I just canât use in my situation, but one of the devs seems actively hostile around providing help to headless servers, so I didnât bother asking forge for help. I thought Iâd write up what I had to do to get things working. As a warning, it requires some local work; you canât do everything headless with these directions.

Iâm running Minecraft 1.6.4, with the latest version of forge for that, 9.11.1.965.

Locally, download and start the minecraft client for the correct version at least once. Not sure if youâll need to âplay onlineâ or not. If you have the current installer, you need to make a new profile with the correct minecraft version and play that.
Copy ~/.minecraft/libraries to the headless machine.
DownloadÂ forge (the installerÂ version, not the universal) fromÂ http://files.minecraftforge.net/.Â The non-adly version is the little star for non-interactive use.

Run -

java -jar forge-1.6.4-9.11.1.965-installer.jar --installServer

Delete the installer, you donât need it any more.
Install any mods you want to the âmodsâ directory, edit server.properties, etc. Normal server setup.
To execute the server, run the file indicated in the installer. In my case, I run -
```
java -jarÂ minecraftforge-universal-1.6.4-9.11.1.965-v164-pregradle.jar nogui
```
-

Alternatively, you can install the entire server locally and copy it over.

-
-
-Susan Tatun says: -
-
- July 10, 2015 at 11:31 pm
-
-
An honest sharing about downloaing and installing Minecraft 1.6.4. I followed what you mentioned and did it sucessfully. Right now, Iâm playing with my little son and guiding him what the terrific things are. Anyway, thanks a lot!
-
-
-Reply
-
-
-
-
-Lynx says: -
-
- August 18, 2015 at 1:44 am
-
-
4am installs of servers is hard, and trying to follow the Forge wiki which is inaccurate at best is hard.
-You got me from dead brick to running box in ten minutes. Thank you.
-
-
-Reply
-
-
-
-
-crumpuppet says: -
-
- September 7, 2015 at 1:32 pm
-
-
Thanks so much! Iâve been looking for these steps for a while, and finally found something that works. Would have been first prize if it could be used along with a GUI frontend like mcmyadmin, but oh well ð
-
-
-Reply
-
-
-
-
-Dave says: -
-
- November 9, 2015 at 11:14 am
-
-
Same here. Searched all over found yours and running in moments. My kids have been hooked on mods and wanted a server. I have VMware and can spin up a linux box up in moments. This was so easy. Thanks.
-
-
-Reply
-
-
-
-
-Minecraft Lover says: -
-
- February 6, 2019 at 6:40 am
-
-
Very useful information, Thank you
-
-
-Reply
-
-
-
-
-Leon says: -
-
- September 9, 2019 at 9:12 am
-
-
Does anyone here have problems with installing the mods. Im donwloading them in the mods directory with the cmd âwgetâ and nothing works.Am i using the wrong command or what?
-
-
-Reply
-
-
-
-
-Rajan Chopra says: -
-
- March 7, 2020 at 3:45 pm
-
-
Thanks for sharing Minecraft. Can you also share Roblox Apk?
-
-
-Reply
-
-
-
-
-Neckbeard Hater says: -
-
- January 4, 2021 at 10:22 pm
-
-
âone of the devs seems actively hostile around providing help to â¦â
-
this is so typical of the linux community. really. And I am a developer with 20 years experience, I hate the Linux community.
-
-
-Reply
-
-
-
-
-nat says: -
-
- February 14, 2023 at 9:17 am
-
-
lol this tutorial still works 9 years later thanks tho this helped so much!!
-
-
-Reply
-
-

I scan each and every piece of paper that passes through my hands. All my old to-do lists, bills people send me in the mail, the manual for my microwave, everything. I have a lot of scans.

scan-organizerÂ is a tool I wrote to help me neatly organize and label everything, and make it searchable. Itâs designed for going through a huge backlog by hand over the course of weeks, and then dumping a new set of raw scans in whenever afterwards. I have a specific processing pipeline discussed below. However if you have even a little programming skill, Iâve designed this to be modified to suit your own workflow.

Input and output

The input is some raw scans. They could be handwritten notes, printed computer documents, photos, or whatever.

The final product is that for each file likeÂ ticket.jpg, we end up withÂ ticket.txt. This has metadata about the file (tags, category, notes) and a transcription of any text in the image, to make it searchable with grep & co.

---
-category: movie tickets
-filename: seven psychopaths ticket.jpg
-tags:
-- cleaned
-- categorized
-- named
-- hand_transcribe
-- transcribed
-- verified
----
-Rialto Cinemas Elmwood
-SEVEN PSYCHOPAT
-R
-Sun Oct 28 1
-7:15 PM
-Adult $10.50
-00504-3102812185308
-
-Rialto Cinemas Gift Cards
-Perfect For Movie Lovers!
-

Here are some screenshots of the process. Apologizies if theyâre a little big! I just took actual screenshots.

At any point I can exit the program, and all progress is saved. I have 6000 photos in the backlogâthis isnât going to be a one-session thing for me! Also, everything has keyboard shortcuts, which I prefer.

Phase 1: Rotating and Cropping

First, I clean up the images. Crop them, rotate them if theyâre not facing the right way. I can rotate images with keyboard shortcuts, although there are also buttons at the bottom. Once Iâm done, I press a button, andÂ scan-organizerÂ advanced to the next un-cleaned photo.

Phase 2: Sorting into folders

Next, I sort things into folders, or âcategoriesâ. As I browse folders, I can preview whatâs already in that folder.

Phase 3: Renaming Images

Renaming images comes next. For convenience, I can browse existing images in the folder, to help name everything in a standard way.

Phase 4: Tagging images

I tag my images with the type of text. They might be handwritten. Or they might be printed computer documents. You can imagine extending the process with other types of tagging for your use case.

Not yet done: OCR

Printed documents are run through OCR. This isnât actually done yet, but it will be easy to plug in. I will probably use tesseract.

Phase 5: Transcribing by hand

I write up all my handwritten documents. I have not found any useful handwriting recognition software. I just do it all by hand.

The point of scan-organizer is to filter based on tags. So only images Iâve marked as needing hand transcription are shown in this phase.

Phase 6: Verification

Â At the end of the whole process, I verify that each image looks good, and is correctly tagged and transcribed.

I wrote a small scheme interpreter in C.

I donât usually like IDEs. Theyâre hard to switch off of, they do too much. They donât let me customize things, and I always have to use external tools anyway. Iâd really rather do things with a bunch of small tools, the linux way. The problem is, if I close everything, Iâll have trouble getting started back up again. Saving state is one solution. Quick start-up is another. Basically, write a checklist for myself to make starting things up easy (open such-and-such files in the editor, start the server in debug mode, etc).

But weâre programmers, so obviously weâre not going to use a literal checklist. Instead, weâre going to write a little script to auto-start things in a new screen session:

#!/usr/bin/screen -c
-# game_development.screen.conf
-# Run stand-alone or with screen -c game_devel.screen.conf
-screen -t "Vim" 2 bash -c "vim -p *.t"
-bind "r" screen -t "Game" 2 bash run.sh

Or if you prefer tmux:

# game_development.tmux.conf
-# Run with tmux -f game_development.tmux.conf attach
-new-session -s game_development
-new-window -n "Vim" "bash -c 'vim -p *.t'"
-bind r new-window -n "Game" "bash run.sh"

Note the main features being used: a shebang line hack for screen, to let this file be self-contained and executable. Opening files in vim in place of a text editor. Binding keys for unit tests, running the program, restarting the server, etc. Now, a similar approach is to add new key bindings to the text editor, but I feel like text editors should edit text, and I like being able to document all the additions with help menus (which screen and tmux both support).

Note: ratpoison is similar to screen/tmux so you can do similar things in X.

One thing Iâd love is if this kind of file was easy to dumpÂ from the current state, especially for things like positioning windows, etc. A little assistance is available, but not too much. Ratpoison and tmux let you dump sizing information. Nothing outputs keybindings or a list of running programs with their windows.

There is a program called tmuxinator to let you write the same config in nested YAML of sessions, panes, and windows, which might appeal to some users.

Also, check out dtach if you donât need panes and windows, and just want a detachable process.

Infocom introduced (AFAIK) the concept of feelies:

-
[â¦] Imaginative props and extras tied to the gameâs themeâprovidedÂ copy protectionÂ againstÂ copyright infringement.^[45]Â Some games were unsolvable without the extra content provided with the boxed game. And because of the cleverness and uniqueness of the feelies, users rarely felt like they were an intrusion or inconvenience, as was the case with most of the other copy-protection schemes of the time.^[49]Â Feelies also provided the player with a physical aspect to the gameplay of their text adventures, giving another dimension of strategy to what would other-wise just be a text parser.
-â Wikipedia (Infocom)

I love to give out feelies for my D&D campaigns. Here are some lil handout props I made:

I used a receipt printer, q-tips, tape, and orthodontic rubber bands.

Generate an SSL/TLS key, which will be used to actually encrypt traffic. -

DOMAIN=nntp.za3k.com
-openssl genrsa -out ${DOMAIN}.key 4096
-chmod 700 ${DOMAIN}.key

Generate a Certificate Signing Request, which is sent to your authentication provider. The details here will have to match the details they have on file (for StartSSL, just the domain name). -

# -subj "/C=US/ST=/L=/O=/CN=${DOMAIN}" can be omitted to fill in custom identification details
-# -sha512 is the hash of your key used for identification. This was the reasonable option in Oct 2014. It isn't supported by IE6
-openssl req -new -key ${DOMAIN}.key -out ${DOMAIN}.csr -subj "/C=US/ST=/L=/O=/CN=${DOMAIN}" -sha512
-

Submit your Certificate Signing Request to your authentication provider. Assuming the signing request details match whatever they know about you, theyâll return you a certificate. You should also make sure to grab any intermediate and root certificates here. -
```
echo "Saved certificate" > ${DOMAIN}.crt
-wget https://www.startssl.com/certs/sca.server1.crt https://www.startssl.com/certs/ca.crt # Intermediate and root certificate for StartSSL
-
```
-
Combine the chain of trust (key, CSR, certificate, intermediate certificates(s), root certificate) into a single file with concatenation. Leaving out the key will give you a combined certificate of trust for the key, which you may need for other applications. -
```
cat ${DOMAIN}.crt sca.server1.crt >${DOMAIN}.pem # Main cert
-cat ${DOMAIN}.key ${DOMAIN}.crt sca.server1.crt ca.crt >${DOMAIN}.full.pem
-chmod 700 ${DOMAIN}.full.pem
```
-

I added a software section to my website. It lists all the software Iâve made over the years (well not all of it, but what I think is most useful to others).

I updated the archive page as well, mostly to remove the software that was there. As usual, I try to make sure all links to my website are good forever.

I decided I wanted to show (restricted) data views on the web in table form. Specifically,Â âstylish.dbâ is aÂ database provided by a chrome plugin. Hereâs an example script, stylish.view, which displays the contents of that. It contains a commentÂ saying which database itâs a query on, together with the query.

-- stylish.db
-SELECT style, code, GROUP_CONCAT(section_meta.value) as 'website(s)' FROM
- (SELECT styles.name AS style,
- sections.code AS code,sections.id AS sections_id
- FROM styles INNER JOIN sections ON sections.style_id = styles.id)
-LEFT JOIN section_meta
- ON section_meta.section_id = sections_id
-GROUP BY style;

The cool part here is that none of this was specific to stylish. I can quickly throw together a .view file for any database and put it on the web.

I add put any databases in cgi-bin/db, and add view.cgi to cgi-bin:

#!/bin/bash
-# view.cgi
-echo "Content-type: text/html"
-echo
-
-QUERY_FILE="${PATH_TRANSLATED}"
-DB_NAME=$(head -n1 "${QUERY_FILE}" | sed -e 's/--\s*//')
-DB="/home/za3k/cgi-bin/db/${DB_NAME}"
-
-echo "<html><head><title>Query on #{DB_NAME}</title><link rel="stylesheet" type="text/css" href="db.css"></head><body><table id=\"${DB_NAME}\">"
-sqlite3 "$DB" -html -header <"${QUERY_FILE}"
-echo "</table></body></html>"

I add this to apacheâs `.htaccess`:

Action view /cgi-bin/view.cgi
-AddHandler view .view
-

-Recipe is mostly from Tricia.

Ingredients:

8-10 oz fresh steak, with salt and pepper
2 tsp capers and/orÂ 2 Tbsp diced olives
3 tsp brown mustard
4 tsp olive oil
2 eggs
2 Tbsp finely diced, fresh red onions
Crushed red pepper to taste

Remove all fat and tendons from the steak. Season it lightly with salt and pepper, and sear lightly on high heat to make it safe to eat. Slice theÂ meat into very thin (2mm) strips, arrange in two piles. Coat the meat in olive oil. Push a small divot into each pile.
Dice olives and onions. Add capers, mustard, and red pepper. Mix together and pour into meat piles equally, or surround the meat with it.
Separate whites and yolks (carefully removing all the white since weâre using raw yolks). Pour one egg yolk into each divot.

Read about raw beef and egg safety first to be well informed.

-
-
-Tricia says: -
-
- May 21, 2015 at 8:14 pm
-
-
So tasty! Enjoyed our month of obsessing over the recipe ^u^
-
-
-Reply
-
-

I did a survey of the cost of buying hard drives (of all sorts), CDs, DVDs, Blue-rays, and tape media (for tape drives).

Here are the 2019-07 results: https://za3k.com/archive/storage-2019-07.sc.txt
2018-10: https://za3k.com/archive/storage-2018-10.sc.txt
2018-06: https://za3k.com/archive/storage-2017-06.sc.txt
2018-01: https://za3k.com/archive/storage-2017-01.sc.txt

I did a survey of the cost of buying hard drives (of all sorts), CDs, DVDs, Blue-rays, and tape media (for tape drives).

Here are the 2020-01 results: https://za3k.com/archive/storage-2020-01.sc.txt
2019-07:Â https://za3k.com/archive/storage-2019-07.sc.txt
2018-10:Â https://za3k.com/archive/storage-2018-10.sc.txt
2018-06:Â https://za3k.com/archive/storage-2017-06.sc.txt
2018-01:Â https://za3k.com/archive/storage-2017-01.sc.txt

Changes this year

I excluded Seagate drives (except where theyâre the only drives in class)
Amazonâs search got much worse, and they started having listings for refurbished drives
Corrected paper archival density, added photographic film
Added SSDs (both 2.5â³ and M.2 formats)
Prices did not go up or down significantly in the last 6 months.

Some conclusions that are useful to know

The cheapest option is tape media, but tape reader/writers for LTO 6, 7, and 8 are very expensive.
The second-cheapest option is to buy external hard drives, and then open the cases and take out the hard drives. This gives you reliable drives with no warrantee.
Blu-ray and DVD are more expensive than buying hard drives

I did a survey of the cost of buying hard drives (of all sorts), microsd/sd, USB sticks, CDs, DVDs, Blu-rays, and tape media (for tape drives).

Here are the 2022-07 results: https://za3k.com/archive/storage-2022-07.sc.txt

2020-01:Â https://za3k.com/archive/storage-2020-01.sc.txt
2019-07:Â https://za3k.com/archive/storage-2019-07.sc.txt
2018-10:Â https://za3k.com/archive/storage-2018-10.sc.txt
2018-06:Â https://za3k.com/archive/storage-2017-06.sc.txt
2018-01:Â https://za3k.com/archive/storage-2017-01.sc.txt

Useful conclusions:

Used or refurbished items were excluded. Multi-packs (5 USB sticks) were excluded except for optical media. Seagate drives were excluded, because they are infamous for having a high failure rate and bad returns process.
Per TB, the cheapest options are:
- Tape media (LTO-8) at $4.74/TB, but I recommend against it. Tape drives are expensive ($3300 for LTO-8 new), giving a breakeven with HDDs at 350-400TB. Also, the world is down to only one tape drive manufacturer, so you could end up screwed in the future.
- 3.5â³ internal spinning hard drives, at $13.75/TB. Currently the best option is 4TB drives.
- 3.5â³ external spinning hard drives, at $17.00/TB. Currently the best is 18TB WD drives. If you want internal drives, you can buy external ones and open them up, although it voids your warranty.
- 2.5â³ external spinning hard drives, at $24.50/TB. 4-5TB is best.
- Blu-ray disks, at $23.16: 25GB is cheapest, then 50GB ($32.38/TB), then 100GB ($54.72/TB).
Be very careful buying internal hard drives online, and try to use a first-party seller. There are a lot of fake sellers and sellers who donât actually provide a warranty. This is new in the last few years.

Changes since the last survey 2 years ago:

Amazonâs search got much worse again. More sponsored listings, still refurbished drives.
Sketchy third-party sellers are showing up on Amazon, and other vendors. At this point the problem is people not getting what they order, or getting it but without a promised warranty. I tried to filter out such Amazon sellers. I had trouble, even though I do the survey by hand. At this point it would be hard to safely buy an internal hard drive on Amazon.
Spinning drives: Prices have not significantly dropped or risen for spinning hard drives, since 2020.
Spinning drives: 18TB and 20TB 3.5â³ hard drives became available
SSDs: 8TB is available (in both 2.5 inch and M.2 formats)
SSDs: Prices dropped by about half, per TB. The cheapest overall drives dropped about 30%.
USB: 2TB dropped back off the market, and appears unavailable.
USB: On the lower end, USB prices rose almost 2X. On the higher end, they dropped.
MicroSD/SD: Prices dropped
MicroSD/SD: A new player entered the cheap-end flash market, TEAMGROUP. Based on reading reviews, they make real drives, and sell them cheaper than they were available before. Complaints of buffer issues or problems with sustained write speeds are common.
MicroSD/SD: Itâs no longer possible to buy slow microsd/sd cards, which is good. Basically everything is class 10 and above.
MicroSD/SD: Combine microsd and sd to show price comparison
Optical: Mostly optical prices did not change. 100GB Blu-Ray dropped by 60-70%. Archival Blu-Ray, too.
Tape: LTO-9 is available.
Tape: The cost of LTO-8 tape dropped 50%, which makes it the cheapest option.
Tape: This is not new, but there is still only one tape drive manufacturer (HP) since around the introduction of LTO-8.

I did a survey of the cost of buying hard drives (of all sorts), microsd/sd, USB sticks, CDs, DVDs, Blu-rays, and tape media (for tape drives).

I excluded used/refurbished options. Multi-packs (5 USB sticks) were excluded, except for optical media like CD-ROMs. Seagate drives were excluded because Seagate has a poor reputation.

Here are theÂ 2023-01 results:Â https://za3k.com/archive/storage-2023-01.sc.txt

2022-07: https://za3k.com/archive/storage-2022-07.sc.txt
2020-01:Â https://za3k.com/archive/storage-2020-01.sc.txt
2019-07:Â https://za3k.com/archive/storage-2019-07.sc.txt
2018-10:Â https://za3k.com/archive/storage-2018-10.sc.txt
2018-06:Â https://za3k.com/archive/storage-2017-06.sc.txt
2018-01:Â https://za3k.com/archive/storage-2017-01.sc.txt

Per TB, the options are (from cheapest to most expensive):

Tape media (LTO-8) at $4.52/TB, but I recommend against it. A tape drive is about $1,600 (twice that new). Thatâs a breakeven at 150-300TB. Also, the world is down to one tape drive manufacturer, so you may end up screwed in the future.
3.5â³ internal spinning hard drives, at $15.00/TB. Currently the best option is 8TB drives.
Optical media, at $16.71/TB. 25GB blu-ray disks are cheapest.
3.5â³ external hard drives, at $17.75/TB. Currently the best option is 18TB drives.
2.5â³ portable spinning hard drives, at $22.00/TB. Currently the best option is 5TB drives.
SSD drives, at $42-$46/TB. Best option is 1TB.
USB sticks, at $59/TB. Best option is 128GB sticks.
MicroSD cards, at $62/TB. Best option is 512GB cards.

Changes since the last survey (4 months ago):

Amazonâs search improved. Less refurbished drives and sponsored listings.
Spinning drives: 22TB 3.5â³ drives became available
Spinning drives: Prices for the previous cheapest option (4TB) rose, making 8TB the new cheap option.
SSDs: Prices dropped by about 30%.
MicroSD/SD: Prices dropped slightly.
Optical: The cheapest option (25GB blu-ray) dropped 30%.
Optical: I stopped gathering data on the cost of BR-RE
Tape: LTO-7 tape drives are now available used, halving the break-even point on tape.

I stopped using OBS a while ago for a couple reasonsâthe main one was that it didnât support my video capture card, but I also had issues with it crashing or lagging behind with no clear indication of what it was doing. I ended up switching to ffmpeg for live streaming, because itâs very easy to tell when ffmpeg is lagging behind. OBS uses ffmpeg internally for video. I donât especially recommend this setup, but I thought Iâd document it in case someone canât use a nice GUI setup like OBS or similar.

Iâm prefer less layers, so Iâm still on ALSA. My setup is:

I have one computer, running linux. It runs what Iâm streaming (typically minecraft), and captures everything, encodes everything, and sends it to twitch
Video is captured using libxcb (captures X11 desktop)
Audio is captured using ALSA. My mic is captured directly, while the rest of my desktop audio is sent to a loopback device which acts as a second mic.
Everything is encoded together into one video stream. The video is a Flash video container with x264 video and AAC audio, because thatâs what twitch wants. Hopefully weâll all switch to AV1 soon.
That stream is sent to twitch by ffmpeg
There is no way to pause the stream, do scenes, adjust audio, see audio levels, etc while the stream is going. I just have to adjust program volumes independently.

Hereâs my .asoundrc:

# sudo modprobe snd-aloop is required to set up hw:Loopback
-pcm.!default {
-  type plug
-  slave.pcm {
-    type dmix
-    ipc_key 99
-    slave {
-      pcm "hw:Loopback,0"
-      rate 48000
-      channels 2
-      period_size 1024
-    }
-  }
-}

My ffmpeg build line:

./configure --enable-libfdk-aac --enable-nonfree --enable-libxcb --enable-indev=alsa --enable-outdev=alsa --prefix=/usr/local --extra-version='za3k' --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --arch=amd64 --enable-gpl --disable-stripping --enable-avresample --disable-filter=resample --enable-avisynth --enable-libaom --enable-libass --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libmp3lame --enable-libopus --enable-libpulse --enable-libvorbis --enable-libvpx --enable-libx265 --enable-opengl --enable-libdrm --enable-libx264 --enable-shared --enable-librtmp && make -j 4 && sudo make install

And most imporantly, my ffmpeg command:

ffmpeg 
-  -video_size 1280x720 -framerate 30 -f x11grab -s 1280x720 -r 30 -i :0.0 
-  -f alsa -ac 1 -ar 48000 -i hw:1,0 
-  -f alsa -ac 2 -ar 48000 -i hw:Loopback,1
-  -filter_complex '[1:a][1:a]amerge=inputs=2[stereo1] ; [2:a][stereo1]amerge=inputs=2[a]' -ac 2 
-  -map '[a]' -map 0:v 
-  -f flv -ac 2 -ar 48000 
-  -vcodec libx264 -g 60 -keyint_min 30 -b:v 3000k -minrate 3000k -maxrate 3000k -pix_fmt yuv420p -s 1280x720 -preset ultrafast -tune film 
-  -c:a libfdk_aac -b:a 160k -strict normal -bufsize 3000k 
-  rtmp://live-sjc.twitch.tv/app/${TWITCH_KEY}

Letâs break that monster down a bit. ffmpeg structures its command line into input streams, transformations, and output streams.

ffmpeg input streams

-video_size 1280x720 -framerate 30 -f x11grab -s 1280x720 -r 30 -i :0.0:
Grab 720p video (-video_size 1280x720) at 30fps (-framerate 30) using x11grab/libxcb (-f x11grab), and we also want to output that video at the same resolution and framerate (-s 1280x720 -r 30). We grab :0.0 (-i :0.0)âthatâs X language for first X server (you only have one, probably), first display/monitor. And, since we donât say otherwise, we grab the whole thing, so the monitor better be 720p.

-f alsa -ac 1 -ar 48000 -i hw:1,0:
Using alsa (-f alsa), capture mono (-ac 1, 1 audio channel) at the standard PC sample rate (-ar 48000, audio rate=48000 Hz). The ALSA device is hw:1,0 (-i hw:1,0), my microphone, which happens to be mono.

-f alsa -ac 2 -ar 48000 -i hw:Loopback,1:
Using alsa (-f alsa), capture stereo (-ac 2, 2 audio channels) at the standard PC sample rate (-ar 48000, audio rate=48000 Hz). The ALSA device is hw:Loopback,1. In the ALSA config file .asoundrc given above, you can see we send all computer audio to hw:Loopback,0. Something sent to play on hw:Loopback,0 is made available to record as hw:Loopback,1, thatâs just the convention for how snd-aloop devices work.

ffmpeg transforms

-filter_complex '[1:a][1:a]amerge=inputs=2[stereo1] ; [2:a][stereo1]amerge=inputs=2[a]' -ac 2:
All right, this one was a bit of a doozy to figure out. In ffmpegâs special filter notation, 1:a means âstream #1, audioâ (where stream #0 is the first one).

First we take the mic input [1:a][1:a] and convert it from a mono channel to stereo, by just duplicating the sound to both ears (amerge=inputs=2[stereo1]). Then, we combine the stereo mic and the stereo computer audio ([2:a][stereo1]) into a single stereo stream using the default mixer (amerge=inputs=2[a]).

-map '[a]' -map 0:v :
By default, ffmpeg just keeps all the streams around, so we now have one mono, and two stereo streams, and it wonât default to picking the last one. So we manually tell it to keep that last audio stream we made (-map '[a]'), and the video stream from the first input (-map 0:v, the only video stream).

ffmpeg output streams

-f flv -ac 2 -ar 48000:
We want the output format to be Flash video (-f flv) with stereo audio (-ac 2) at 48000Hz (-ar 48000). Why do we want that? Because weâre streaming to Twitch and thatâs what Twitch says they wantâthatâs basically why everything in the output format.

-vcodec libx264 -g 60 -keyint_min 30 -b:v3000k -minrate 3000k -maxrate 3000k -pix_fmt yuv420p -s 1280x720 -preset ultrafast -tune film:
Ah, the magic. Now we do x264 encoding (-vcodec libx264), a modern wonder. A lot of the options here are just what Twitch requests. They want keyframes every 2 seconds (-g 60 -keyint_min 30, where 60=30*2=FPS*2, 30=FPS). They want a constant bitrate (-b:v3000k -minrate 3000k -maxrate 3000k) between 1K-6K/s at the time of writingâI picked 3K because itâs appropriate for 720p video, but you could go with 6K for 1080p. Here are Twitchâs recommendations. The pixel format is standard (-pix_gmt yub720p) and we still donât want to change the resolution (-s 1280x720). Finally the options you might want to change. You want to set the preset as high as it will go with your computer keeping upâmine sucks (-preset ultrafast, where the options go ultrafast,superfast,veryfast,faster,fast,medium, with a 2-10X jump in CPU power needed for each step). And Iâm broadcasting minecraft, which in terms of encoders is close to film (-tune film)âlots of panning, relatively complicated stuff on screen. If you want to re-encode cartoons you want something else.

-c:a libfdk_aac -b:a 160k:
We use AAC (-c:a libfdk_aac). Note that libfdk is many times faster than the default implementation, but itâs not available by default in debianâs ffmpeg for (dumb) license reasons. We use 160k bitrate (-b:a 160k ) audio since Iâve found thatâs good, and 96K-160K is Twitchâs allowable range. `-strict normal`

-strict normal: Just an ffmpeg option. Not interesting.
-bufsize 3000k: One second of buffer with CBR video

rtmp://live-sjc.twitch.tv/app/${TWITCH_KEY}:
The twitch streaming URL. Replace ${TWITCH_KEY} with your actual key, of course.

Sources:

jrayhawk on IRC (alsa)
ffmpeg wiki and docs (pretty good)
ALSA docs (not that good)
Twitch documentation, which is pretty good once you can find it
mark hills on how to set up snd-aloop

-
-
-Mario Enriquez says: -
-
- January 22, 2021 at 7:31 pm
-
-
Thanks for your post.
-
-
-Reply
-
-

Just a quick shout-out to Chrome extensionÂ Stylish, which lets you add custom stylesheets to any web page. Iâm using it with âdisplay: noneâ and âvisibility: hiddenâ exclusively, to hide annoying page elements.

Install packages talk, talkd,Â inetutils-inetd
Make sure users have set âmesg yâ
âtalk user@localhostâ. Despite documentation, âtalk userâ will not work.

CFAR usually designs their techniques to help people Get Stuff Done. I have a failure mode of Getting The Wrong Stuff Done, so this time through their workshop, I focused on improving techniques to explicitly have steps around pursuing the correct terminal goals (which Iâll here call âterminal goal techniquesâ).

Original technique: Goal-factor
-New terminal goal technique:

FindÂ an instrumental goal toward another instrumental goal.
Embark on that goal provisionally, while also making a plan to acquire more information about whether itâs a good idea and better plans are available.
Periodically re-evaluate to make sure itâs the best goal and youâre gathering information.

Original Technique: Murphy-jitsu
-

Come up with a plan to achieve your goal.
Use pre-hindsight to ask âHow didÂ this go wrong?â.
Fix that. Go to 2 until fixed.

Revised terminal goal technique: Murphy-jitsu with terminal goal check

Come up with a plan.
Use pre-hindsight to ask âHow wasÂ I disappointed when this wentÂ as planned?â
Fix that. Go to 2 until fixed.
Use pre-hindsight to ask âHow didÂ this go wrong?â.
Fix that. Go to 2 until fixed.

Another technique: Positive murphyjitsu (âWhy didnât this go even better?â)
-Another technique: Aversion murphyjitsu (âImagine none of the positive listed factors happened. Why was it still possible?â) for cases when I canât think about how to overcome a aversion directly.

Instrumental failure TAP:

Notice executive planning fails Â ->Â Look for lack ofÂ motivation or motivation propagation
Notice motivation / motivation propogation fails ->Â Look to see if the goal youâre pursuing is what you want (or exactly what you want)

Technique: Exactboxing

Find a thing to do
Set a 15-minute timer
Do the thing
If you finish early, keep doing the thing anyway. (For example, figure out how to do it better in the future or for the problem to never happen again.
If you donât finish in time, stop anyway. Itâs done. If you donât know how itâs done, thatâs a failure modeâyou should have at most one task which takes an unbounded amount of time in your life. (I find this makes my brain accept âline of retreatâ)

Technique:Â Do the Obvious Thing

Ask a question, for example how to pursue a goal you want achieved (I recommend a Hamming Question)
Figure out theÂ most obvious solution.
Acknowledge that it is the most obvious solution.
Decide whether or not to do it.
If you donât want to, contradiction. Debug steps 1,2,3,4 and see where you went wrong until theyâre in accord.

Theory on how to avoid lost purposes (mostly from Eliezer): Use Litany of Tarski a lot until you get the magic effect where you donât start rationalizing to begin with (and generally donât flinch away from learning about things/mistakes). Then, develop an aversion to lost purposes. The naive failure mode is to avoid noticing lost purposes if you have an aversion. (The simpler technique is Alien in a Body)

-
-
-JSevilla says: -
-
- July 18, 2015 at 9:42 pm
-
-
Cool stuff. I liked exactboxing and DTOT.
-Murphy jitsu has potential.
-The modified goal factor has a suspicious component of leaving things to future me that I donât like a bit though.
-
-
-Reply
-
-

Recently I wrote a scraper. First, I downloaded all the HTML files. Next, I wanted to parse the content. However, real world data is pretty messy. I would run the scraper, and it would get partway though the file and fail. Then I would improve it, and it would get further and fail. Iâd improve it more, and it would finish the whole file, but fail on the fifth one. Then Iâd re-run things, and it would fail on file #52, #1035, and #553,956.

To make testing faster, I added a scaffold. Whenever my parser hit an error, it would print the filename (for me, the tester) and record the filename to an error log. Then, it would immediately exit. When I re-ran the parser, it would test all the files where it had hit a problem first. That way, I didnât have to wait 20 minutes until it got to the failure case.

if __name__ == "__main__":
-    if os.path.exists("failures.log"):
-        # Quicker failures 
-        with open("failures.log", "r") as f:
-            failures = set([x.strip() for x in f])
-        for path in tqdm.tqdm(failures, desc="re-checking known tricky files"):
-            try:
-                with open(path) as input:
-                    parse_file(input)
-            except Exception:
-                print(path, "failed again (already failed once")
-                raise
-
-    paths = []
-    for root, dirs, files in os.walk("html"):
-        for file in sorted(files):
-            path = os.path.join(root, file)
-            paths.append(path)
-    paths.sort()
-
-    with open("output.json", "w") as out:
-        for path in tqdm.tqdm(paths, desc="parse files"): # tqdm is just a progress bar. you can also use 'for path in paths:
-            with open(input, "r") as input:
-                try:
-                    result = parse_file(input)
-                except Exception:
-                    print(path, "failed, adding to quick-fail test list")
-                    with open("failures.log", "a") as fatal:
-                        print(path, file=fatal)
-                    raise
-                json.dump(result, out, sort_keys=True) # my desired output is one JSON dict per line
-                out.write("\n")

-
Abbot: I will perform the opening prayer in the New Latin. Oh ordlay, ivethgay usway ouryay essingsblay. Amen-ay!
-
Crowd: AMEN-AY!
-
âRobin Hood: Men In Tights
-

In honor of National Novel Writing/GeneratingÂ Month and Christmas spirit, I translated the King James Bible into the âNew Latinâ (aka Pig Latin).

1:1 Inway ethay eginningbay Odgay eatedcray ethay eavenhay andway ethay earthway.
-
-1:2 Andway ethay earthway asway ithoutway ormfay, andway oidvay; andway arknessday asway uponway
-ethay acefay ofway ethay eepday. Andway ethay Iritspay ofway Odgay ovedmay uponway ethay acefay ofway ethay
-atersway.
-
--Ethay Iblebay

The full bible translation is here. Source code is here.

Two forces pull at me: the desire to have few possessions and be able to travel flexibly, and the convenience of reading and referencing physical books. I discovered a third option: I have digital copies of all my books, so I can freely get rid of them at any time, or travel without inconvenience.

So thatâs where we start. Hereâs where I went.

I thought, if these books are just a local convenience for an online version, itâs more artistically satisfying to have some representation of that. So I printed up a card catalog of all my books, both the ones I have digital copies of and not:

An example catalog card

Thatâs what a card looks like. Thereâs information about the book up top, and a link in the form of a QR code in the middle. The link downloads a PDF version of that book. Obviously being aÂ programmer, the cards all all automatically generated.

Book with a card inside

For the books where I have a physical copy, I put the card in the book, and it feels like Iâm touching the digital copy. My friends can pirate their own personal version of the book (saving me the sadness of lost lent-out books Iâm sure weâve all felt at times). And I just thing it looks darn neat.Â Some physical books I donât have a digital version of, since the world is not yet perfect. But at least I can identify them at a glance (and consider sending them off to a service likeÂ http://1dollarscan.com/)

Card catalog of digital books

And then, I have a box full of all the books I *donât* have a physical copy of, so I can browse through them, and organize them into reading lists or recommendations. Itâs not nearly as cool as the ones in books, but itâs sort of nice to keep around.

And if I ever decide to get rid of a book, I can just check to make sure thereâs a card inside, and move the card into the box, reassured nothing is lost, giving away a physical artifact I no longer have the ability to support.

I sadly wonât provide a link to the library since that stuff is mostly pirated.

Interesting technical problems encountered during this project (you can stop reading now if youâre not technically inclined):

Making sure each card gets printed exactly once, in the face of printer failures and updating digital collections. This was hard and took up most of my time, but itâs also insanely boring so Iâll say no more.
Command-line QR code generation, especially without generating intermediate files. I used rqrcode_png in ruby. I can now hotlink link qr.png?text=Hello%20World and see any textÂ I want, itâs great.
Printing the cards. This is actually really difficult to automateâI generate the cards in HTML and itâs pretty difficult to print HTML, CSS, and included images. I ended up using the âwkhtmltoimageâ project, which as far as I can tell, renders the image somewhere internally using webkit and screenshots it. Thereâs also a wkhtmltopdf available, which worked well but I couldnât get to cooperate with index-card sized paper. Nothing else really seems to handle CSS, etc properly and as horrifying as the fundamental approach is, itâs both correct and well-executed. (They solved a number of problems with upstream patches to Qt for example, the sort of thing I love to hear)
The zbarcam software (for scanning QR codes among other digital codes) is just absolute quality work and I canât say enough good things about it. Scanning cards back into the computer was one of the most pleasant parts of this whole project. It has an intuitive command UI using all the format options I want, and camera feedback to show itâs scanned QR codes (which it does very quickly).
Future-proofed links to pirated booksâthe sort of link that usually goes down. I opted to use a SHA256 hash (the mysterious numbers at the bottom which form a unique signature generated from the content of the book) and provide a small page on my website which gives you a download based on that. This is what the QR code links to. I was hoping there was some way to provide that without involving me, but Iâm unaware of anyÂ service available. Alice Monday suggested just typing the SHA hash into Google, which sounded like the sort of clever idea which might work. It doesnât.

Summary of âthe life-changing magic of tidying upâ:

Marie Kondo writes the âKonMariâ method. The book ends up being as much about her mistakes inÂ learning how to tidy as it is about how to tidy. The book conveys a certain positive energy that makes me want to recommend it, but the author also brings that energy in reaction to a kind of previous stress which accompanied tidying, which she does not seem to have completely droppedâif you are mysteriously anxious and feel you MUST discard everything after reading her book, this may be why.

The primary point she makes is meant to cure it: Decide what to keep and what to discard by physically touching each item, and asking if it brings you joy.

The rest of the method:

Positivity. Everything in your house loves and wants to help you.Â If it is time to send off some of the items on their next adventure, this is no reason to be sad or anxious. You had a great time meeting, and theyÂ and you were both happy.
Tidy all at once (at least by category, but preferably in a multi-day binge).
Physically gather the category in once place, touching everything and asking if it brings you joy.
Find out what youâll keep and discard before putting things away or organizing.
Organizing: ??? [I didnât get any big takeaways here].

Marie Kondoâs best advice is realizations from her past mistakesâthe sort of methods which seems reasonable to try, but end up being wrong for subtle reasons. They are:

Tidy by category, not place. Otherwise, you wonât realize everything you have.
âStorageâ is storing things neatly, and lets you have more and more things. This is different than tidying, which is about bringing things in harmony, and having only things you love. Becoming better at âstorageâ can make you unhappy.

She also has encountered her clients making mistakes. For each category of things (clothes, books, etc) there are many reasons clients may not want to throw something out. Most of the book is meant to illustrate why these things are useless, and why throwing them out is okay and will make you happier.

The fun part is that many clients were more confident and more in touch with what they valued and who they wanted once they had only possessions they loved.

Bolded text in the book

Start by discarding. Then organize your space, thoroughly, completely, in one go.
A dramatic reorganization of the home causes coorespondingly dramatic changes in lifestyle and perspective. It is life transforming.
when you put your house in order, you put your affairs and your past in order, too
They are surrounded only by the things they love
the magic of tidying
People cannot change their habits without first changing their way of thinking
If you tidy up in one shot, rather than little by little, you can dramatically change your mind-set.
If you use the right method and concentrate your efforts on eliminating clutter thoroughly and completely with a short span of time, youâll see instant results that will empower you to keep your space in order ever after.
Tidying is just a tool, not a final destination. [The true goal should be to establish the lifestyle you want most once your house has been put in order]
A booby trap lies within the term âstorageâ.
Putting things away creates the illusion that the clutter problem has been solved.
Tidying up location by location is a fatal mistake.
Effective tidying involves only two essential actions: discarding and deciding where to store things. Of the two, discarding must come first.
Tidying is a special event. Donât do it every day.
Your goal is clearly in sight. The moment you have put everything in its place, you have crossed the finish line.
Tidy in the right order.
Do not even think of putting your things away until you have finished the process of discarding.
Think in concrete terms so that you can vividly picture what it would be like to live in a clutter-free space.
However, the moment you start focusing on how to choose what to throw away, you have actually veered significantly off course.
We should be choosing what we want to keep, not what we want to get rid of.
take each item in oneâs hand and ask: âDoes this spark joy?â If it does, keep it. If not, dispose of it.
Keep only those things that speak to your heart. Then take the plunge and discard all the rest.
-always think in terms of category, not place
People have trouble discarding things that they could still use (functional value), that contain helpful information (informational value), and that have sentimetnal value). When these things are hard to obtain or replace (rarity), they become even harder to part with.
The best sequence is this: clothes first, then books, papers, komono (miscellany), and lastly, mementos.
itâs extremely stressful for parents to see what their children discard
To quietly work away at disposing of your own excess is actually the best way of dealing with a family that doesnât tidy. The urge to point out someone elseâs failure to tidy is usually a sign that you are neglecting to take care of your own space.
To truly cherish the things that are important to you, you must first discard those that have outlived their purpose.
What things will bring you joy if you keep them a part of your life?
The most important points to remember are these: Make sure you gather every piece of clothing in the house and be sure to handle each one.
By neatly folding your clothes, you can solve almost every problem related to storage.
The key is to store things standing up rather than laid flat.
The goal is to fold each piece of clothing into a simple, smooth rectangle.
Every piece of clothing has its own âsweet spotâ where it feels just right
Arrange your clothes so they rise to the right.
By category, coats would be on the far left, followed by dresses, jackets, pants, skirts, and blouses.
Never, ever tie up your stockings. Never, ever ball up your socks.
The trick is not to overcategorize. Divide your clothes roughly into âcotton-likeâ and âwool-likeâ materials when you put them in the drawer.
If you are planning to buy storage units in the near future, I recommend that you get a set of drawers instead.
The criterion is, of course, whether or not it gives you a thrill of pleasure when you touch it.
In the end, you are going to read very few of your books again.
The moment you first encounter a particular book is the right time to read it.
[Papers] I recommend you dispose of anything that does not fall into one of three categories: currently in use, needed for a limited period of time, or must be kept indefinitely.
[Papers that need to be dealt with] Make sure that you keep all such papers in one spot only. Never let them spread to other parts of the house.
[On lecture/seminar papers] Itâs paradoxical, but I believe precisely because we hang on to such materials, we fail to put what we learn into practice.
Too many people live surrounded by things they donât need âjust becauseâ.
Presents are not âthingsâ but a means for conveying someoneâs feelings.
Mysterious cords will always remain just thatâa mystery.
Despite the fact that coins are perfectly good cash, they are treated with far less respect than paper money.
No matter how wonderful things used to be, we cannot live in the past. The joy and excitement we feel here and now are more important.
People never retrieve the boxes they send âhomeâ. Once sent, they will never again be opened.
By handling each sentimental item and deciding what to discard, you process your past.
As you reduce your belongings through the process of tidying, you will come to a point where you suddenly know how much is just right for you.
The fact that you possess a surplus of things that you canât bring yourself to discard doesnât mean you are taking good care of them. In fact, it is quite the opposite.
Believe what your heart tells you when you ask, âDoes this spark joy?â
The point in deciding specific places to keep things is to designate a spot for every thing.
Once you learn to choose your belongings properly, you will be left only with the amount that fits perfectly in the space you currently own.
pursue ultimate simplicity in storage
I have only two rules: store all items of the same type in the same place and donât scatter storage space.
If you live with your family, first clearly define separate storage spaces for each family member.
Everyone needs a sanctuary
Clutter is caused by a failure to return things to where they belong. Therefore, storage should reduce the effort needed to put things away, not the effort needed to get them out.
If you are aiming for an uncluttered room, it is much more important to arrange your storage so that you can tell at a glance where everything is than to worry about the details of who does what, where, and when.
When you are choosing what to keep, ask your heart; when you are choosing where to store something, ask your house.
stacking is very hard on the things at the bottom
Rather than buying something to make do for now, wait until you have completed the entire process and then take your time looking for storage items that you really like.
The key is to put the same type of bags together.
One theme underlying my method of tidying is transforming the home into a sacred place, a power spot filled with pure energy.
Transform your closet into your own private space, one that gives you a thrill of pleasure.
Stockings take up 25 percent less room once they are out of the package and folded up.
By eliminating excess visual information that doesnât inspire joy, you can make your space much more peaceful and comfortable.
[homework assignment to clients] appreciate their belongings [by actually expressing appreciation to them]
At their core, the things we really like do not change over time. Putting your house in order is a great way to discover what they are.
letting go is even more important than adding
The lives of those who tidy thoroughly and completely, in a single shot, are without exception dramatically altered.
one of the magical effects of tidying is confidence in your decision-making capacity
But when we really delve into the reasons for why we canât let something go, there are only two: an attachment to the past or a fear for the future.
The question of what you want to own is actually the question of how you want to live your life.
The sum total of all the garbage so far would exceed twenty-eight thousand bags, and the number of items discarded must be over one million.
The fact that they do not need to search is actually an invaluable stress reliever..
Life becomes far easier once you know that things will still work out even if you are lacking something.
I believe that tidying is a celebration, a special send-off for those things that will be departing from the house, and therefore I dress accordingly.
In essence, tidying ought to be the act of restoring balance among people, their possessions, and the house they live in.
Make your parting a ceremony to launch them on a new journey.
Itâs a very strange phenomenon, but when we reduce what we own and essentially âdetoxâ our house, it has a detox effect on our bodies as well.
If you can say without a doubt, âI really life this!â no matter what anyone else says, and if you like yourself for having it, then ignore what other people think.
As for you, pour your time and passion into what brings you the most joy, your mission in life.

I write down everything I do.

I transcribed my journals by hand. That is, I typed them up myself, instead of trying to use handwriting recognition or outsourcing to Mechanical Turk.

I started on 2019-11-02, and finished today, 2020-11-20. Thatâs roughly one year.
The 15 journals transcribed go from 2011 to 2020, 10 years. The 2011-2015 period is sparser.
Of the 15 journals, 13 of them them I transcribed from the physical version. Two I had thrown out, because my old scanner was feed-through, and I had to destroy the spines to scan the books.
Thatâs 1779 pages total (small ones, these are pocket journals). Itâs also 32,000 lines, and 164K words. The text is 1.1MB, the scanned PNG files are 12GB (12000 MB).
In general, it takes me 1 hour to transcribe the last week of notes. Going farther back is harder, partly because my handwriting gets more readable as time progresses (due at least as much to my choice of pen, as my neatness), and partly because I have a harder time guessing at poor handwriting without memory to fill it in, and partly because I didnât use standard formats back then.
I do have exact numbers I could check, but a lower bound based on this rate is that was overall 90 hours of work. It probably didnât take more than twice that.

I want to talk about three different mindsetsÂ for time management and what constitutes âsuccessâ. In all three, Iâll talk about an example problem, letâs say âwriting term papersâ for a student who finds this difficult, and see how each might approach the goal.Â My goal is not to promote one of these as best, but rather to illustrate that all three exist. Each may be favored by one person or another, or by a single person depending on the situation. I hope that by describing them, I can help people understand one anotherâsÂ motivations, so as to facilitate communication. The first mindsetÂ I will call optimization. The optimizer tries to spend their time gaining resources. For example, they might work to get money. They also attempt to increase the rateÂ at which they can gain resources. Some optimizers even try to increase the rate at which they can e.g. network or learn skills. The intuition here is that most goods are somehow fungible, and that you should try to get as many fungible goods as possible. Example of term papers: An optimizer might try to learn to write term papers, or get faster and faster at writing papers. If they got good at writing term papers, they might try to write even more (for example, taking classes heavy on papers) to take advantage of these skills. Heuristics:

Get nowhere faster: Get where youâre going faster, even if you donât have a specific goal in mind
Level up: Itâs always good to learn things and develop skills
Experiment: Never be content with the status quo. (Choose explore/exploit explicitly and dovetail, for principled experimenters)
Accumulate multipliers / Seek good trade routes:Â Be able to buy all things with money, learning, whatever you have most of. Try to maximize the rate.
Get money: Anywhere you have a resource coming in, increase the incomeÂ rate, or trade rate youâre getting it for.

Failure modes:

Butterfly: Ends up unfocused and with no immediate or terminal goals.

The second mindsetÂ I will call satisficing. The satisficer is goal-oriented, and tries to stay focus on the goal at all times. TraditionalÂ goal-based time management like GTD encourages satisficing. Example of term papers: Their specific goal might be âGet an A on all papers until I graduateâ. If they can write well enough to get an A, they probably would not learn to write better. If they canât get an A, their goal might be better served by learning to write to rubrics. The question of learning to write better vs spending more time editing on each paper would be decided by the amount of time each would take for the actual number of term papers the student expected to write for their immediate goal. Heuristics:

Get to the goal as quickly as possible
No lost purposes: One of the strengths of satisficing is to avoid ârabbit holesâ which donât contribute to any end goal
Munchkin: Think about what will actually contribute to the goal, as well as what wonât.

Failure modes:

Tunnel vision / inflexibility: Not contributing to the immediate goal is not the same as being useless. Can lose sight of the big picture (supergoal or unrelated terminal goals) and ways to work toward that other than via the immediate goal.
EA Bot Syndrome / Donât smell the flowers: Excessive goal focus can lose sight of human involvement, and end up giving the satisficer low life quality.

The last mindsetÂ I will call minimization. The minimizer will try to minimize the amount of resources spent on a particular task or problem. They will especially try to avoid any indefinitely-big cost. Example of term papers: The student might try to spend the minimum amount of time possible on the paper to get an A. If they knew they were going to write several papers, they might study how to write papers faster, so as to spend less total timeâeven if they expected the time learning to write faster would be greater than the time to write the immediate papers (âbut what if there are more papers somedayâ). The eventual state of the minimizer will probably be to have set things up to avoid term papers, or write them very quickly. Heuristics:

Automate / Offload for free: Spend only a finite amount of time on something
Eliminate recurring costs
Asymptotic improvement: Any finite number of mistakes is okay, as long as you end up optimal eventually
Deal with it permanently: If youâre not closing things and returning to them indefinitely, youâre spending infinite time and attention on it
Timebox: Only spend a finite amount of time on something, and then consider it resolved forever

Failure modes:

Bad at goals: Minimization deals with recurring activities, but can fail to offer any positive motivation for one-time end goals
Negative mindset: Heavy focus on mistakes and resource use

-
-
-Tricia says: -
-
- November 5, 2015 at 9:13 pm
-
-
Very nice! I often find myself automatically optimizing â the easiest place to see this is in Animal Crossing where Iâll keep a large cash of items for when villagers might ask for one. The idea is to always be prepared (I blame girl scouts). I suppose the negative to this behavior is that it could lead to hording if left unchecked.
-
-
-Reply
-
-

I write down everything I do. Yesterday, I wrote a quick-and-dirty analysis program to get some stats on common habits.

The full results are here: d r ive floss food read sleep teeth tv wake walk youtube. Of course, what I write down doesnât perfectly match what I do, so most of the absolute stats are vastly wrong. Comparative ones are still interesting.

Hereâs some results:

The data say Iâm a night owl. I go to sleep 0.9 times a day on average. 2am-4am is most common. I wake up at 11am-12pm.
I brush my teeth 0.18 times a day, on average, but 0.36 times in 2021. Surely thatâs not right!? I have flossed 25 times since 2018. Iâm going be honestâ¦ that one probably is about right, yikes.
I watch 30% more youtube on Tuesday than Saturday, but only 4% more television. I watch most TV between 9pm and 1am. Youtube is pretty spread out. I watch tv 0.7 times a day, buy youtube 1.9 times a day.
I read 1.1 times a day, usually before midnight.
I eat 3.5 times a day on average. This includes 796 days where I didnât record eating. Omitting those, I eat 6.5 times a day.

Iâll leave you with this poor manâs graph of driving frequency over time. See the pandemic hit?

{'2018-01': 'x',
- '2018-02': '',
- '2018-03': 'x',
- '2018-04': 'xxx',
- '2018-05': '',
- '2018-06': '',
- '2018-07': 'x',
- '2018-08': '',
- '2018-09': '',
- '2018-10': 'x',
- '2018-11': 'xx',
- '2018-12': 'xx',
- '2019-01': 'x',
- '2019-02': '',
- '2019-03': 'xxxxxxx',
- '2019-04': 'x',
- '2019-05': '',
- '2019-06': 'x',
- '2019-07': 'x',
- '2019-08': 'xx',
- '2019-09': '',
- '2019-10': 'xxxxx',
- '2019-11': 'xxxxxxxxx',
- '2019-12': 'xxxxxx',
- '2020-01': 'xxxxxxxxx',
- '2020-02': 'xxxxxxxxxxx',
- '2020-03': 'xxxxxxx',
- '2020-04': 'xxxxx',
- '2020-05': 'xx',
- '2020-06': '',
- '2020-07': '',
- '2020-08': '',
- '2020-09': '',
- '2020-10': 'x',
- '2020-11': 'xxxxxx',
- '2020-12': 'x',
- '2021-01': 'xx',
- '2021-02': 'xxx',
- '2021-03': 'xxxxxxxxx',
- '2021-04': '',
- '2021-05': 'xxxxxxxx',
- '2021-06': 'xxxxxxxxxxxxxxx',
- '2021-07': 'xxxxxxxxxxxxxxxxx',
- '2021-08': 'xxxxx',
- '2021-09': 'xxxxxxx',
- '2021-10': 'xxxx',
- '2021-11': 'xxxxxx',
- '2021-12': 'xxxxxxxx',
- '2022-01': 'xxxxxxxxx',
- '2022-02': 'xxxxxxxxxx',
- '2022-03': 'xxxxxxxxxxxx',
- '2022-04': 'xxxxxxxxxxxxxxxxxxxxxxx',
- '2022-05': 'xxxxxxxxxxxxxxxxxxxxxxx',
- '2022-06': 'xxxxxxxxxxxxxxxxx',
- '2022-07': 'xxxxxxxxxxxxxxxxxxxxxxx',
- '2022-08': 'xxxxxxxxxxxxxxxxxxxxxxx',
- '2022-09': 'xxxxxxx'}

Made using Piskell

-
-
-Tricia says: -
-
- October 14, 2015 at 10:03 am
-
-
Hola! Just wanted to let you know I nominated you for the Versatile Blogger Award, for more info check out my post here: http://sometimesicook.net/2015/10/14/the-versatile-blogger-award/ ð
-
-
-Reply
-
-

I recently wrote a program that records all tty activity. That means bash sessions, ssh, raw tty access, screen and tmux sessions, the lot. I used script. The latest version of my software can be found on github.

Note that itâs been tested only with bash so far, and thereâs no encryption built in.

To just record all shell commands typed, use the standard eternal history tricks (bash).

Letâs take a look at the gzip format. Why might you want to do this?

Maybe youâre curious how gzip works
Maybe youâre curious how DEFLATE works. DEFLATE is the âactualâ compression method inside of gzip. Itâs also used in zip, png, git, http, pdfâ¦ the list is pretty long.
Maybe you would like to write a gzip/DEFLATE decompressor. (A compressor is more complicatedâunderstanding the format alone isnât enough)

Letâs work a few examples and look at the format in close detail. For all these examples, Iâm using GNU gzip 1.10-3 on an x86_64 machine.

I recommend checking out the linked resources below for a deeper conceptual overview if you want to learn more. That said, these are the only worked examples of gzip and/or DEFLATE of which Iâm aware, so theyâre a great companion to one another. In particular, you may want to learn what a prefix code is ahead of time.

References:
[1] RFC 1951, DEFLATE standard, by Peter Deutsch
[2] RFC 1952, gzip standard, by Peter Deutsch
[3] infgen, by Mark Adler (one of the zlib/gzip/DEFLATE authors), a tool for dis-assembling and printing a gzip or DEFLATE stream. I found this useful in figuring out the endian-ness of bitfields, and somewhat in understanding the dynamic huffman decoding process. Documentation is here.
[4] An explanation of the âdeflateâ algorithm by Antaeus Feldspar. A great conceptual overview of LZ77 and Huffman coding. I recommend reading this before reading my DEFLATE explanation.
[5] LZ77 compression, Wikipedia.
[6] Prefix-free codes generally and Huffmanâs algorithm specifically
[7] After writing this, I learned about puff.c, a reference (simple) implementation of a DEFLATE decompressor by Mark Adler.

Gzip format: Basics and compressing a stream

Letâs take a look at our first example. If youâre on Linux, feel free to run the examples I use as we go.

echo "hello hello hello hello" | gzip

The bytes gzip outputs are below. You can use xxd or any other hex dump tool to view binary files. Notice that the original is 24 bytes, while the compressed version is 29 bytesâgzip is not really intended for data this short, so all of the examples in this article actually get bigger.

Byte	0	1	2	3	4	5	6	7	8	9	10	11	12	13	14	15	16	17	18	19	20	21	22	23	24	25	26	27	28
Hex	1f	8b	08	00	00	00	00	00	00	03	cb	48	cd	c9	c9	57	c8	40	27	b9	00	00	88	59	0b	18	00	00	00

hello (1) â gzip contents

The beginning and end in bold are the gzip header and footer. I learned the details of the format by reading RFC 1952: gzip

Byte 0+1 (1f8b): Two fixed bytes that indicate âthis is a gzip fileâ. These file-type indicators are also called âmagic bytesâ.
Byte 2 (08): Indicates âthe compression format is DEFLATEâ. DEFLATE is the only format supported by gzip.
Byte 3 (00): Flags. 8 single-bit flags.
- Not set: TEXT (indicates this is ASCII text. hint to the decompressor only. i think gzip never sets this flag)
- Not set: HCRC (adds a 16-bit CRC to the header)
- Not set: EXTRA (adds an âextrasâ field to the header)
- Not set: NAME (adds a filename to the headerâif you compress a file instead of stdin this will be set)
- Not set: COMMENT (adds a comment to the header)
- There are also three reserved bits which are not used.
Byte 4-7 (00000000): Mtime. These indicate when the compressed file was last modified, as a unix timestamp. gzip doesnât set an associated time when compressing stdin. Technically the standard says it should use the current time, but this makes the output the same every time you run gzip, so itâs better than the original standard.
Byte 8 (00): Extra flags. 8 more single-bit flags, this time specific to the DEFLATE format. None are set so letâs skip it. All they can indicate is âminimum compression levelâ and âmax compression levelâ.
Byte 9 (03): OS. OS â03â is Unix.
Byte 10-20: Compressed (DEFLATE) contents. Weâll take a detailed look at DEFLATE below.
Byte 21-24 (0088590b): CRC32 of the uncompressed data, âhello hello hello hello\nâ. I assume this is correct. Itâs worth noting, there are multiple things called âCRC32â.
Byte 25-28 (18000000): Size of the uncompressed data. This is little-endian byte order, 0x00000018 = 16*1+1*8 = 24. The uncompressed text is 24 bytes, so this is correct.

Byte	10	11	12	13	14	15	16	17	18	19	20
Hex	cb	48	cd	c9	c9	57	c8	40	27	b9	00
Binary	11001011	01001000	11001101	11001001	11001001	01010111	11001000	01000000	00100111	10111001	00000000
R. Bin.	11010011	00010010	10110011	10010011	10010011	11101010	00010011	00000010	11100100	10011101	00000000

hello (1) â DEFLATE contents

DEFLATE format: Basics and fixed huffman coding

DEFLATE is the actual compression format used inside gzip. The format is detailed in RFC 1951: DEFLATE. DEFLATE is a dense format which uses bits instead of bytes, so we need to take a look at the binary, not the hex, and things will not be byte-aligned. The endian-ness is a little confusing in gzip, so weâll usually be looking at the âreversed binaryâ row.

As a hint, whenever we read bits, we use the âreverseâ binary order. For Huffman codes, we keep the bit order in reverse. For fixed-length fields like integers, we reverse again into ânormalâ binary order. Iâll call out the order for each field.
Byte 10: 1 1010011. Is it the last block? Yes.
- 1: Last block. The last block flag here means that after this block ends, the DEFLATE stream is over
Byte 10: 1 10 10011. Fixed huffman coding. We reverse the bits (because itâs always 2 bits, and we reverse any fixed number of bits) to get 01.
- 00: Not compressed
- 01: Fixed huffman coding.
- 10: Dynamic huffman coding.
- 11: Not allowed (error)
So weâre using âfixedâ huffman coding. That means thereâs a static, fixed encoding scheme being used, defined by the DEFLATE standard. The scheme is given by the tables below. Note that Length/Distance codes are specialâafter you read one, you may read some extra bits according to the length/distance lookup tables.

Binary	Bits	Extra bits	Type	Code
00110000-10111111	8	0	Literal byte	0-143
110010000-111111111	9	0	Literal byte	144-255
0000000	7	0	End of block	256
0000001-0010111	7	varies	Length	257-279
11000000-11000111	8	varies	Length	280-285

Literal/End of Block/Length Huffman codes

Binary Code	Bits	Extra bits	Type	Value
00000-111111	5	varies	Distance	0-31

Distance Huffman codes

Code	Binary	Meaning	Extra bits
267	0001011	Length 15-16	1

Length lookup (abridged)

Code	Binary	Meaning	Extra bits
4	00100	Distance 5-6	1

Distance lookup (abridged)

Now we read a series of codes. Each code might be
- a literal (one binary byte), which is directly copied to the output
- âend of blockâ. either another block is read, or if this was the last block, DEFLATE stops.
- a length-distance pair. first code is a length, then a distance is read. then some of the output is copiedâthis reduces the size of repetitive content. the compressor/decompressor can look up to 32KB backwards for duplicate content. This copying scheme is called LZ77.
Huffman codes are a âprefix-free codeâ (confusingly also called a âprefix codeâ). What that means is that, even though the code words are different lengths from one another, you can always unambigously tell which binary codeword is next. For example, suppose the bits youâre reading starts with: 0101. Is the next binary codeword 0, 01, 010, or 0101? In a prefix-free code, only one of those is a valid codeword, so itâs easy to tell. You donât need any special separator to tell you the codeword is over. The space savings from not having a separator is really important for good compression. The âhuffmanâ codes used by DEFLATE are prefix-free codes, but theyâre not really optimal Huffman codesâitâs a common misnomer.
Byte 10-11: 110 10011000 10010: A literal. 10011000 (152) minus 00110000 (48) is 104. 104 in ASCII is âhâ.
Byte 11-12: 000 10010101 10011: A literal. 10010101 (149) minus 00110000 (48) is 101. 101 in ASCII is âeâ.
Byte 12-13: 101 10011100 10011: A literal. 10011100 (156) minus 00110000 (48) is 108. 108 in ASCII is âlâ.
Byte 13-14: 100 10011100 10011: Another literal âlâ
Byte 14-15: 100 10011111 01010: A literal. 10011111 (159) minus 00110000 (48) is 111. 111 in ASCII is âoâ.
Byte 15-16: 111 01010000 10011: A literal. 01010000 (80) minus 00110000 (48) is 32. 32 in ASCII is â â (space).
Byte 16-17: 000 10011000 00010: Another literal âhâ.
Byte 17: 000 0001011: A length. 0001011 (11) minus 0000001 (1) is 10, plus 257 is 267. We look up distance 256 in the âlength lookupâ table. The length is 15-16, a range.
Byte 18: 1 00100: Because the length is a range, we read extra bits. The âlength lookupâ table says to read 1 extra bit: 1. The extra bits need to be re-flipped back to normal binary order to decode them, but 0b1 flipped is still 0b1. 15 (bottom of range) plus 0b1 = 1 (extra bits) is 16, so the final length is 16.
Byte 18-19: 111 00100 10011101: After a length, we always read a distance next. Distances are encoded using a second huffman table. 00100 is code 4, which using the âdistance lookupâ table is distance 5-6.
Byte 18-19: 11100100 1 0011101. Using the âdistance lookupâ table, we need to read 1 extra bit: 0b1. Again, we reverse it, and add 5 (bottom end of range) to 0b1 (extra bits read), to get a distance of 6.
We copy from 6 characters ago in the output stream. The stream so far is âhello hâ, so 6 characters back is starting at âeâ. We copy 16 characters, resulting in âhello hello hello helloâ. Why this copy didnât start with the second âhâ instead of the second âeâ, Iâm not sure.
Byte 19-20: 1 00111010 0000000: A literal. 00111010 (58) minus 00110000 (48) is 10. 10 in ASCII is â\nâ (new line)
Byte 20: 0 0000000: End of block. In this case we ended nicely on the block boundry, too. This is the final block, so weâre done decoding entirely.
At this point weâd check the CRC32 and length match whatâs in the gzip footer right after the block.

Our final output is âhello hello hello hello\nâ, which is exactly what we expected.

Gzip format: Compressing a file

Letâs generate a second example using a file.

echo -en "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8\xf7\xf6\xf5\xf4\xf3\xf2\xf1" >test.bin
-gzip test.bin

This input file is pretty weird. In fact, itâs so weird that gzip compression will fail to reduce its size at all. Weâll take a look at what happens when compression fails in the next DEFLATE section below. But first, letâs see how gzip changes with a file instead of a stdin stream.

Byte	0	1	2	3	4	5	6	7	8	9	10	11	12	13	14	15	16	17	18	19-38	39	40	41	42	43	44	45	46
Hex	1f	8b	08	08	9f	08	ea	60	00	03	74	65	73	74	2e	62	69	6e	00	see below	c6	d3	15	7e	0f	00	00	00

binary garbage (2) â abridged gzip contents

Okay, letâs take a look at how the header and footer changed.

Byte 0+1 (1f8b): Two fixed bytes that indicate âthis is a gzip fileâ. These file-type indicators are also called âmagic bytesâ.
Byte 2 (08): Indicates âthe compression format is DEFLATEâ. DEFLATE is the only format supported by gzip.
Byte 3 (08): Flags. 8 single-bit flags.
- Not set: TEXT (indicates this is ASCII text. hint to the decompressor only. i think gzip never sets this flag)
- Not set: HCRC (adds a 16-bit CRC to the header)
- Not set: EXTRA (adds an âextrasâ field to the header)
- Set: NAME (adds a filename to the header)
- Not set: COMMENT (adds a comment to the header)
- There are also three reserved bits which are not used.
Byte 4-7 (9f08ea60) . Mtime. This is in little-endian order: 0x60ea089f is 1625950367. This is a unix timestamp â 1625950367 seconds after midnight, Jan 1, 1970 is 2021-07-10 20:52:47 UTC, which is indeed earlier today. This is the time the original file was last changed, not when compression happened. This is here so we can restore the original modification time if we want.
Byte 8 (00): Extra flags. None are set.
Byte 9 (03): OS. OS â03â is Unix.
Byte 10-18 (74 65 73 74 2e 62 69 6e 00): Zero-terminated string. The string is âtest.binâ, the name of the file to decompress. We know this field is present because of the flag set.
Byte 19-38: The compressed DEFLATE stream.
Byte 39-42 (c6d3157e): CRC32 of the uncompressed data. Again, Iâll just assume this is correct.
Byte 25-28 (0f000000): Size of the uncompressed data. 0x0000000f = 15 bytes, which is correct.

DEFLATE format: Uncompressed data

Uncompressed data is fairly rare in the wild from what Iâve seen, but for the sake of completeness weâll cover it.

Byte	19	20	21	22	23	24-38
Hex	01	0f	00	f0	ff	ff fe fd fc fa f9 f8 f7 f6 f5 f4 f3 f2 f1
Binary	00000001	00001111	00000000	11110000	11111111	omitted
R. Binary	10000000	11110000	00000000	00001111	11111111	omitted

binary garbage (2) â DEFLATE contents

Again, we start reading âr. binaryâ â the binary bits in reversed order.
Byte 19: 10000000. The first three bits are the most important bits in the stream:
- 1: Last block. The last block flag here means that after this block ends, the DEFLATE stream is over
Byte 19: 100 00000. Not compressed. For a non-compressed block only, we also skip until the end of the byte.
- 00: Not compressed
- 01: Fixed huffman coding.
- 10: Dynamic huffman coding.
Byte 20-21: 11110000 00000000. Copy 15 uncompressed bytes. We reverse the binary bits as usual for fixed fields. 0b0000000000001111 = 0x000f = 15.
Byte 22-23: 00001111 11111111. This is just the NOT (compliment) of byte 20-21 as a check. It can be ignored.
Byte 24-38: ff fe fd fc fb fa f9 f8 f7 f6 f5 f4 f3 f2 f1: 15 literal bytes of data, which are directly copied to the decompressed output with no processing. Since we only have one block, this is the whole of the decompressed data.

DEFLATE format: Dynamic huffman coding

Dynamic huffman coding is by far the most complicated part of the DEFLATE and gzip specs. It also shows up a lot in practice, so we need to learn this too. Letâs take a look with a third and final example.

echo -n "abaabbbabaababbaababaaaabaaabbbbbaa" | gzip

The bytes we get are:

Byte 0-9 (1f 8b 08 00 00 00 00 00 00 03): Header
Byte 10-32 (1d c6 49 01 00 00 10 40 c0 ac a3 7f 88 3d 3c 20 2a 97 9d 37 5e 1d 0c): DEFLATE contents
Byte 33-40 (6e 29 34 94 23 00 00 00): Footer. The uncompressed data is 35 bytes.

Weâve already seen everything interesting in the gzip format, so weâll skip the header and footer, and move straight to looking at DEFLATE this time.

Byte	10	11	12	13	14	15	16	17	18	19	20	21	22	23	24	25	26	27	28	29	30	31	32
Hex	1d	c6	49	01	00	00	10	40	c0	ac	a3	7f	88	3d	3c	20	2a	97	9d	37	5e	1d	0c
Binary	00011101	11000110	01001001	00000001	00000000	00000000	00010000	01000000	11000000	10101100	10100011	01111111	10001000	00111101	00111100	00100000	00101010	10010111	10011101	00110111	01011110	00011101	00001100
R. Binary	10111000	01100011	10010010	10000000	00000000	00000000	10000000	00000010	00000011	00110101	11000101	11111110	00010001	10111100	00111100	00000100	01010100	11101001	10111001	11101100	01111010	10111000	00110000

abaa stream â DEFLATE contents

As usual, we read âr. binaryâ â the binary bits in reversed order.
Byte 10: 10111000. Last (only) block. The DEFLATE stream is over after this block.
Byte 10: 10111000. 10=Dynamic huffman coding
- 00: Not compressed
- 01: Fixed huffman coding.
- 10: Dynamic huffman coding.

Which parts are dynamic?

Okay, so what does âdynamicâ huffman coding mean? A fixed huffman code had several hardcoded values defined by the spec. Some are still hardcoded, but some will now be defined by the gzip file.

The available literals are all single-byte literals. The literals remain fixed by the spec.
There is a âspecialâ literal indicating the end of the block in both.
The lengths (how far to look backwards when copying) were given as ranges whose size was a power of two. For example, there would be one binary code (0001011) for the length range 15-16. Then, we would read one extra bit (because the range is 2^1 elements long) to find the specific length within that range. In a dynamic coding, the length ranges remain fixed by the spec. (âlength lookupâ table)
Again, the actual ranges and literals are fixed by the spec. The binary codewords to represent (lengths/literals/end-of-block) are defined in the gzip stream instead of hardcoded. (âliteral/end-of-block/length huffman codesâ table)
Like the literal ranges, the distance ranges remain fixed by the spec. (âdistance lookupâ table)
Although the distance ranges themselves are fixed, the binary codewords to represent distance ranges are defined in the gzip stream instead of hardcoded. (âdistance huffman codesâ table)

So basically, the possible lengths and distances are still the same (fixed) ranges, and the literals are still the same fixed literals. But where we had two hardcoded tables before, now we will load these two tables from the file. Since storing a table is bulky, the DEFLATE authors heavily compressed the representation of the tables, which is why dynamic huffman coding is so complicated.

Aside: Storing Prefix-Free Codewords as a List of Lengths

Suppose we have a set of prefix-free codewords: 0, 10, 1100, 1101, 1110, 1111. Forget about what each codeword means for a second, weâre just going to look at the codewords themselves.

We can store the lengths as a list: 1, 2, 4, 4, 4, 4.

You could make another set of codewords with the same list of lengths. But for our purposes, as long as each value gets the same length of codeword, we donât really care which of those codes we pickâthe compressed content will be the same length.
Since we donât really care how if the bits change, any code is fine. For simplicity, we pick a unique âstandardâ code. When we list the codewords, the standard one can be listed BOTH in order of length, AND in normal sorted order. That is, the those two orders are the same. The example code above is a standard code. Hereâs one that isnât: 1, 01, 00.
It turns out that if we have the lengths, we can generate a set of prefix-free codewords with those lengths. Thereâs an easy algorithm to generate the âstandardâ code from the list of lengths (see RFC 1951, itâs not very interesting)
Since we picked the standard codewords, we can switch back and forth between codewords and codeword lengths without losing any information.
Itâs more compact to store the codeword lengths than the actual codewords. DEFLATE just stores codeword lengths everywhere (and uses the corresponding generated code).

Finally, we need to make them correspond to symbols, so we actually store

We store lengths for each symbol: A=4, B=1, C=4, D=4, E=2, F=4
We can get the correct codewords by going through the symbols in order, and grabbing the first available standard codeword: A=1100, B=0, C=1101, D=1110, E=10, F=1111.

Dynamic Huffman: Code Lengths

Whatâs a âcode lengthâ? Itâs yet another hardcoded lookup table, which explains how to compress the dynamic huffman code tree itself. Weâll get to it in a secondâthe important thing about it for now is that there are 19 rows in the table. The binary column (not yet filled in) is what weâre about to decode.

Binary	Code	What it means	Extra bits
?	0-15	Code length 0-15	0
?	16	Copy the previous code length 3-6 times	2
?	17	Copy â0â code length 3-10 times	3
?	18	Copy â0â code length 11-138 times	7

Code Lengths (static)

Byte 10: 101 11000. Number of literal/end-of-block/length codes (257-286). Read bits in forward order, 0b00011=3, plus 257 is 260 length/end-of-block/literal codes.
Byte 11: 01100 011. Number of distance codes (1-32). Read bits in forward order, 0b00110=6, plus 1 is 7 distance codes.
Byte 11-12: 01100 0111 0010010. Number of code length codes used (4-19). Read bits in forward order, 0b1110=14, plus 4 is 18.
Byte 12-18 1 001 001 010 000 000 000 000 000 000 000 000 001 000 000 000 100 000 001 1: There are 18 codes used (out of 19 available). For each code length code, we read a 3-bit number (in big-endian order) called the âcode length code lengthâ, and fill the end with 0s: 4, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 4[, 0]
Next, we re-order the codes in the order 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15. This re-order is just part of the spec, donât ask whyâitâs to save a small amount of space.
The old order is: 16: 4, 17: 4, 18: 2, 0:0, 8:0, 7:0, 9:0, 6:0, 10:0, 5:0, 11:0, 4:4, 12:0, 3:0, 13:0, 2:1, 14:0, 1:4, 15:0
The new order is: 0:0, 1:4, 2:1, 3:0, 4:4, 5:0, 6:0, 7:0, 8:0, 9:0, 10:0, 11:0, 12:0, 13:0, 14:0, 15:0, 16: 4, 17: 4, 18: 2
0s indicate the row is not used (no codeword needed). Letâs re-write it without those.
1:4, 2:1, 4:4, 16: 4, 17: 4, 18: 2
Now we assign a binary codewords of length N, to each length N in the list.
1:1100,2:0,4:1101,16:1110,17:1111,18:10
Finally, letâs take a look at the whole table again.

Binary	Code	What it means	Extra bits
1100	1	Code length 1	0
0	2	Code length 2	0
1101	4	Code length 4	0
1110	16	Copy the previous code length 3-6 times	2
1111	17	Copy â0â code length 3-10 times	3
10	18	Copy â0â code length 11-138 times	7

Code Lengths

Great, weâve parsed the code lengths table.

Dynamic Huffman: Parsing the Huffman tree

As a reminder, in bytes 10-12 we found there was a 260-row literal/end-of-block/length table and a 7-row distance table. Letâs read 267 numbers: the lengths of the codeword for each row.
Byte 18-19: 0000001 10 0110101. Copy â0â code length 11-138 times
0b1010110=86, plus 11 is 97. Literals 0-96 are not present.
Byte 20: 1100 0101: Literal 1. Literal 97 (âaâ) has a codeword of length 1.
Byte 20: 1100 0 101: Literal 2. Number 98 (âbâ) has a codeword of length 2.
Byte 20-21: 11000 10 1111111 10. Copy â0â code length 11-138 times. 0b1111111=127, plus 11 is 138. Literals 99-236 are not present.
Byte 21-22: 111111 10 0001000 1. Copy â0â code length 11-138 times. 0b0001000=8, plus 11 is 19. Literals 237-255 are not present.
Bytes 22-23: 0001000 1101 11100. Literal 256 (end-of-block) has a codeword of length 4.
Byte 23-24: 101 1110 00 0111100. Copy previous code 3-6 times. 0b00=0, plus 3 is 3. âLiteralsâ 257-259 (all lengths) have codewords of length 4.
We read 260 numbers, thatâs the whole literal/end-of-block/length table. Assign the âstandardâ binary codewords based on the lengths to generate the following table:

Literal Code	Code Length	Binary	Meaning
97	1	0	Literal âaâ
98	2	10	Literal âbâ
256	4	1100	End-of-block
257	4	1101	Length 3
258	4	1110	Length 4
259	4	1111	Length 5

abaa dynamic literal/end-of-block/length Huffman codes

Now we read 7 more numbers in the same format: the 7-row distances table.
Byte 24: 0 0 111100. Distance 0 has a codeword of length 2.
Byte 24-25: 00 1111 000 0000100. Copy â0â code length 3-10 times. 0b000=0, plus 3 is 3. Distances 1-3 are not present.
Byte 25: 0 0 0 0 0100: Distances 4-6 have length 2.
We read 7 numbers, thatâs the whole distances table. Assign the âstandardâ binary codewords to generate the following table:

Code	Bits	Binary	Meaning	Extra Bits
0	2	00	Distance 1	0
4	2	01	Distance 5-6	1
5	2	10	Distance 7-8	1
6	2	11	Distance 9-12	2

abaa dynamic literal/end-of-block/length Huffman codes

Dynamic Huffman: Data stream decoding

Now weâre ready to actually decode the data. Again, weâre reading a series of codes from the literal/end-of-block/length Huffman code table.
Byte 25: 00000 10 0: Literal âaâ, âbâ, âaâ
Byte 26: 0 10 10 10 0: Literal âaâ, âbâ, âbâ, âbâ, âaâ.
Byte 27: 1110 10 0 1. Length 4. Whenever we read a length, we read a distance. The distance is a range, 7-8. The extra bit we read is 0b0=0, plus 7 is Distance 7. So we look back 7 bytes and copy 4. The new output is: baabbbabaab
Byte 27-28: 1110100 1101 11 00 1: Length 3, Distance 9. We look back 9 bytes and copy 3. The new output is: abbabaababb
Byte 28-29: 1011100 1111 01 1 00. Length 5, Distance 6. We look back 6 bytes and copy 5. The new output is: aababbaabab
Byte 29: 111011 0 0. Literal âaâ, âaâ.
Byte 30: 0 1111010. Literal âaâ.
Byte 30: 0 1111 01 0. Length 5, Distance 5. We look back 5 bytes and copy 5. The new output is: abaaaabaaa
Byte 31: 10 111000: Literal âbâ
Byte 31: 10 1110 00: Length 4, Distance 1. We look back 1 byte and copy 4. The new output is: bbbbb
Byte 32: 0 0 110000: Literal âaâ, âaâ.
Byte 32: 00 1100 00: End-of block. Since this is the final block itâs also the end of the stream. This didnât come up in the first example, but we zero-pad until the end of the byte when the block ends.
The final output is a b a a b b b a baab abb aabab a a a abaaa b bbbb a a (spaces added for clarity), which is exactly what we expected.

-
-
-Lee says: -
-
- July 23, 2021 at 1:47 pm
-
-
âNow we assign a binary codewords of length N, to each length N in the list.
-1:1100,2:0,4:1101,16:1110,17:1111,18:10â
-
Thereâs no explanation as to where you pulled these bits from, I understand the number of bits comes from the aforementioned 414442 pulled out prior to this but Iâve seen nothing that explains how you pulled out the bit values for each of these code words
-
-
-Reply
-
-
- -
  -
  -admin says: -
  -
  - July 23, 2021 at 2:04 pm
  -
  -
  This is what âAside: Storing Prefix-Free Codewords as a List of Lengthsâ is about. The algorithm is given in full in RFC 1952 if youâre interested. Basically, the code words are assigned starting from 0 (or 00, 000, etc) and counting up. As each code is assigned, anything with that code as a prefix becomes unavailable. They are assigned first from shortest to longest, and in the case of ties from leftmost in the list to rightmost in the list.
  -
  So first we assign 0 to the length-1 codeword (0 is lower than 1), then we assign the length-2 codeword 10 (the lowest length-2 code that doesnât have a prefix 0), and finally we assign the length-4 codewords from left to right (1100, 1101, 1110, 1111 donât have a prefix of 0 or 10 â they are the lowest and only codewords without one of those prefixes).
  -
  Thanks for the comment, it makes me happy to know anyone is reading this, and itâs very helpful to know whatâs unclear in my writeup. Iâd love to make it as good as possible.
  -
  -
  -Reply
  -
  -
  - -
    -
    -Lee says: -
    -
    - July 23, 2021 at 3:23 pm
    -
    -
    I think I get it now, as soon as I resolve some compile errors Iâm gonna try this:
    -
    â`
    - while ( cur_leng < max_leng )
    - {
    - for ( uint j = 0; j leng > cur_leng )
    - continue;
    
    -
    word->code = code;
    -
    code++;
    - }
    -
    while ( !(code >> cur_leng) )
    - ++code;
    -
    ++cur_leng;
    - }
    -â`
    
    -
    -
    -Reply
    -
    -
  -
-
-
-
-Lee says: -
-
- July 25, 2021 at 12:06 pm
-
-
Took me a while to understand that I was supposed to be building the tree at this point (although this method compared to what I though of and implemented prior to this is inefficient both in space and speed, also terribly um-intuitive). Iâm having trouble understanding exactly how to extract some literals youâve mentioned:
-
âByte 18-19: 0000001 10 0110101. Copy â0â code length 11-138 times
-0b1010110=86, plus 11 is 97. Literals 0-96 are not present.
-Byte 20: 1100 0101: Literal 1. Literal 97 (âaâ) has a codeword of length 1.
-Byte 20: 1100 0 101: Literal 2. Number 98 (âbâ) has a codeword of length 2.â

-
At first I though to get the literals I was supposed to add the code list position to the copy count but then I saw this after:
-
âByte 20-21: 11000 10 1111111 10. Copy â0â code length 11-138 times. 0b1111111=127, plus 11 is 138. Literals 99-236 are not present.
-Byte 21-22: 111111 10 0001000 1. Copy â0â code length 11-138 times. 0b0001000=8, plus 11 is 19. Literals 237-255 are not presentâ
-
Which doesnât follow that pattern, how do you determine the literals range?
-
-
-Reply
-
-
- -
  -
  -admin says: -
  -
  - July 25, 2021 at 12:27 pm
  -
  -
  Iâm glad this was helpful, but you should really go read RFC 1951 if youâre going to build a decoder. This isnât meant to be a stand-alone guide.
  -
  Iâm pretty lost trying to follow this comment, sorry. To clarify, for the âdynamicâ compression, you first extract the CODING TABLE for the literals (the step youâre talking about). Then, you use the coding table to decompress the actual stream of content. Itâs a two-step process, which is why itâs so complex.
  -
  The key point I suspect youâre missed was: âLetâs read 267 numbers: the lengths of the codeword for each row.â These numbers are the length of the code for each literal in order (all of them, not just some of them). A length of 0 means ânot presentâ.
  -
  -
  -Reply
  -
  -
  - -
    -
    -Lee says: -
    -
    - July 25, 2021 at 1:41 pm
    -
    -
    Was gonna say I already had the code table but when it printed it seems the codes got corrupted, Iâll have to get back to you after fixing it. Anyways I find âguidesâ that donât use real examples as this one does to be confusing sometimes, usually where it matters, thatâs why I was consulting you who provided those key examples, I got a heck of a lot further with your guide than any other Iâve tried following which didnât give the byte by byte, bit by bit example.
    -
    -
    -Reply
    -
    -
    - -
      -
      -Lee says: -
      -
      - July 25, 2021 at 1:50 pm
      -
      -
      Turned out I was just printing the wrong value, hereâs my current output:
      -
      `
      -./a.out aba.gz
      -path = âaba.gzâ
      -PrintStreamDetails( 0x7ffd2cf70d80 ): ptr = 0x556653fbc480, pos = 0, num = 0, max = 328, fed = 0
      -PrintBytes( 0x556653fbc480, 41, 16 )
      -1F 8B 0 0 0 0 0 0 0 3 1D C6 49 1 0 0
      -40 C0 AC A3 7F 88 3D 3C 20 2A 97 9D 37 5E 1D C
      -29 34 94 23 0 0 0
      -PrintStreamDetails( 0x7ffd2cf70d80 ): ptr = 0x556653fbc480, pos = 136, num = 80, max = 328, fed = 56
      -gzip.magic = 1F8B, gzip.format = 0, gzip.flags = 0, gzip.mtime = 0, zlib.xflags = 0, zlib.system = 3
      -gzip.flag_TEXT = false, gzip.flag_HCRC = false, gzip.flag_MORE = false, gzip.flag_NAME = false, gzip.flag_NOTE = false, gzip.flag_RESERVED = 0
      -PrintStreamDetails( 0x7ffd2cf70d80 ): ptr = 0x556653fbc480, pos = 136, num = 80, max = 328, fed = 56
      -last = true, type = 2
      -lengc = 260, distc = 7, codec = 18, left = 60
      -pos = 212, max = 328, count = 267
      -left = 61, byte = 18, bit = 7
      -Code Table:
      -_list[ 0]: from = 1, more = 0, leng = 4, copy = 0, code = 1100
      -_list[ 1]: from = 2, more = 0, leng = 1, copy = 0, code = 0
      -_list[ 2]: from = 4, more = 0, leng = 4, copy = 0, code = 1101
      -_list[ 3]: from = 16, more = 2, leng = 4, copy = 3, code = 1110
      -_list[ 4]: from = 17, more = 3, leng = 4, copy = 3, code = 1111
      -_list[ 5]: from = 18, more = 7, leng = 2, copy = 11, code = 10
      -i = 0010 j = 5, from = 18, copy = 97
      -Expecting character âaâ
      -Expecting character âbâ
      -i = 0010 j = 5, from = 18, copy = 138
      -i = 0010 j = 5, from = 18, copy = 19
      -Expecting character â
      -i = 1110 j = 3, from = 16, copy = 3
      -Expecting character â
      -i = 1111 j = 4, from = 17, copy = 3
      -Expecting character â
      -Expecting character â
      -Expecting character â
      -PrintBytes( 0x556653fbf4a0, 0, 16 )
      -`
      -
      And hereâs the currently unfixed loop producing the bottom part of the output (also where the interpretation ends atm)
      -
      `
      -/* Should be building a tree here. */
      - while ( num < count && stream.num < stream.max )
      - {
      - CODEWORD *word = NULL;
      - uint j = 0;
      -
      for ( cur_leng = 1; cur_leng <= max_leng; ++cur_leng )
      - {
      - i = RevBits( CopyStreamBits( &stream, cur_leng, false ), cur_leng );
      -
      for ( j = 0; j leng == cur_leng && word->code == i )
      - {
      - IncStreamCount( &stream, cur_leng );
      - break;
      - }
      - }
      -
      if ( j max_leng )
      - {
      - printf
      - (
      - âi = %u, j = %u, list.used = %u, cur_leng = %u\nâ,
      - i, j, list.used, cur_leng
      - );
      - return Return( ret, EINVAL );
      - }
      -
      size = CopyStreamBits( &stream, word->more, true );
      -
      if ( word->from >= 16 )
      - {
      - printf( âi = â );
      - SeeBits( &i, max_leng );
      - printf
      - (
      - â j = %u, from = %2u, copy = %u\nâ,
      - j, word->from, (uint)(size + word->copy)
      - );
      - num += min_literal_code = size + word->copy;
      - }
      - else
      - {
      - int c = (int)(min_literal_code + j);
      - printf( âExpecting character â%câ\nâ, c );
      - symbol = symbols + num;
      - symbol->sym = (char)c;
      - symbol->len = j;
      - ++num;
      - }
      - }
      -`
      -
      -
      -Reply
      -
      -
      - -
        -
        -Lee says: -
        -
        - July 25, 2021 at 3:32 pm
        -
        -
        Never min, I think I know where the literals are coming from now, I did this:
        -
        -printf( "Expecting character '%c', ", c ); -printf( "num as a character = '%c'\n", num ); -
        -
        To see if num lined up, since it did Iâm guessing those ranges where all the wasted leaves/branches of the huffman tree.
        -
        -
        -
        -
        -
      -
    -
  -
-
-
-
-Lee says: -
-
- July 26, 2021 at 7:46 am
-
-
I think I still misunderstood something somewhere, Iâve done a detailed post on the section I think Iâve mis-interpreted here:
-
https://cboard.cprogramming.com/c-programming/180462-increment-gone-wrong.html#post1302477
-
The forum there is better able to handle pre-formatted code than this comment system so please take a look and tell me if you think Iâve mis-interpreted something also, and if so what you think is being mis-interpreted, in mean time Iâm going to try and clean up my code so I can upload to gitlab so that you can a clearer idea of where Iâm at and perhaps help me get to the last part, converting huffman codes to original values
-
-
-Reply
-
-
- -
  -
  -admin says: -
  -
  - July 26, 2021 at 8:54 am
  -
  -
  Sorry, I donât really want to help someone else debug code or output. Iâd suggest following the links at the start of the blog article. Check out âinfgenâ in particular.
  -
  -
  -Reply
  -
  -
  - -
    -
    -Lee says: -
    -
    - July 26, 2021 at 9:34 am
    -
    -
    I originally gave up on infgen due to an access rights error when I tried installing via the package manager, the server refused to let me download it, then after your last comment I though to look for itâs github page, that one worked out fine, judging by itâs output the only thing Iâve got wrong are the length values for the distance symbols/codes/whatever you want to call them, where do you get that 2 from? same applies to the bit code along side them, where does that come from, is it just an iterated number for each valid symbol?
    -
    -
    -Reply
    -
    -
    - -
      -
      -admin says: -
      -
      - July 26, 2021 at 10:56 am
      -
      -
      You read 267=260+7 codeword-lengths (the numbers 260 and 7 are given by bytes 10-12 in this example).
      -
      The first 260 codeword-lengths are used for one huffman table: 256 literals (always 256), then 1 âend of blockâ (always 2), then the remaining 3 are lengths.
      -
      The next 7 codeword-lengths are used to generate a second huffman table, for distances. The code generation works exactly the same as for the first table.
      -
      -
      -Reply
      -
      -
      - -
        -
        -Lee says: -
        -
        - July 26, 2021 at 11:50 am
        -
        -
        Even if I use the same method of code generation as before I stiil would need to know where you got those â2âs from in the table immediately following this:
        -
        âWe read 7 numbers, thatâs the whole distances table. Assign the âstandardâ binary codewords to generate the following table:â
        -
        Because the previous method used that to determine what symbols to ignore until the code length increases, more specifically I had a âwhile ( cur_leng < max_leng )" loop and a sub loop that went looking for symbols expecting cur_leng codes and assigning them that way before using those same codes in another sub loop to decide if the next code should be increased further. Without knowing where those code lengths for the distance came from I can't progress.
        -
        Thinking about it now I don't even have the codes for the length symbols, I just have the symbols. If it was supposed to be auto generated then I could understand the codes for the length symbols but I then don't understand the codes for the distance codes as they don't follow the huffman principle of no codes that can be misinterpreted.
        -
        -
        -
        -
        -
      - -
        -
        -admin says: -
        -
        - July 26, 2021 at 12:23 pm
        -
        -
        Re-read above the table. The âbitsâ column is from bytes 24-25.
        -
        -
        -
        -
        -
      -
    -
  -
-
-
-
-Lee says: -
-
- July 26, 2021 at 12:43 pm
-
-
No option to reply so Iâll do it here instead, I still donât see where the 2 came from, is that a minimum length or from another hard code table? So far I can only see the bits given as a set/unset symbol & code flag, thereâs no clear source for the length & extra bits values
-
-
-Reply
-
-
- -
  -
  -Lee says: -
  -
  - July 26, 2021 at 5:54 pm
  -
  -
  Finally found where you got the 2 from:
  -
  https://www.w3.org/Graphics/PNG/RFC-1951
  -
  â HDIST + 1 code lengths for the distance alphabet,
  -
  encoded using the code length Huffman code
  -â
  -
  That was far too easily over looked, now I at least can generate the codes, though I think I will add a specialised handler and object to minimize code now that itâs getting more complicated than need be.
  -
  -
  -Reply
  -
  -
-
-
-
-Lee says: -
-
- July 31, 2021 at 11:29 am
-
-
Iâve clearly misunderstood something somewhere, could you take a look at the code I outlined in the below post please and see if you can spot what Iâm misunderstanding, up until the point I have to lookup previously deflated values Iâve read the bits correctly but Iâve obviously not understood all the implied information correctly, I would post the code here but as youâve seen these comments donât support maintaining the code formatting resulting in a more confusing than necessary code.
-
https://cboard.cprogramming.com/c-programming/180462-increment-gone-wrong-post1302541.html#post1302541
-
-
-Reply
-
-
- -
  -
  -Lee says: -
  -
  - August 2, 2021 at 5:36 pm
  -
  -
  Youâll be glad to know I finally got the algorithm right, have not looked at any source code from zlib or other projects so Iâm free to slap MIT License on it the whole way through once I convert it to a cleaner version of itself. The unclean version is here for anyoneâs reference.
  -
  https://gitlab.com/awsdert/uc-evidence/-/tree/9441a73e59834456c41c1049036fc60925b705a0
  -
  -
  -Reply
  -
  -
-
-
-
-neubert says: -
-
- December 21, 2021 at 1:00 am
-
-
From the Fixed huffman coding section:
-
âByte 10-11: 110 10011000 10010: A literal. 10011000 (152) minus 00110000 (48) is 104. 104 in ASCII is âhâ.â
-
Why are you subtracting 48?
-
-
-Reply
-
-
- -
  -
  -admin says: -
  -
  - December 21, 2021 at 1:11 am
  -
  -
  The binary range (given above) is 00110000-10111111. Rather than decoding the binary value, we decode the offset within that range.
  -
  -
  -Reply
  -
  -
-
-
-
-neubert says: -
-
- December 22, 2021 at 7:27 am
-
-
âNow we assign a binary codewords of length N, to each length N in the list.
-1:1100,2:0,4:1101,16:1110,17:1111,18:10â
-
Could you elaborate on this? I thought I could get the same thing by using huffman encoding, using the code length as the frequency, but when I do that I get this:
-
1:111,2:1100,4:00,16:10,17:01,18:1101
-
Thanks!
-
-
-Reply
-
-
- -
  -
  -neubert says: -
  -
  - December 22, 2021 at 10:49 pm
  -
  -
  I figured it out. Itâs this bit from RFC1951:
  -
  - 1) Count the number of codes for each code length. Let - bl_count[N] be the number of codes of length N, N >= 1.
  -
  2) Find the numerical value of the smallest code for each
  - code length:
  -
  code = 0;
  - bl_count[0] = 0;
  - for (bits = 1; bits <= MAX_BITS; bits++) {
  - code = (code + bl_count[bits-1]) << 1;
  - next_code[bits] = code;
  - }
  -
  3) Assign numerical values to all codes, using consecutive
  - values for all codes of the same length with the base
  - values determined at step 2. Codes that are never used
  - (which have a bit length of zero) must not be assigned a
  - value.
  -
  for (n = 0; n <= max_code; n++) {
  - len = tree[n].Len;
  - if (len != 0) {
  - tree[n].Code = next_code[len];
  - next_code[len]++;
  - }
  -
  -
  -
  -Reply
  -
  -
-
-
-
-neubert says: -
-
- December 24, 2021 at 10:38 pm
-
-
ââLiteralsâ 257-259 (all lengths) have codewords of length 4â Is this a hard and fast rule thatâs always true regardless of the data or is this true _just_ for the compressed string in this example?
-
-
-Reply
-
-
- -
  -
  -admin says: -
  -
  - December 25, 2021 at 2:25 pm
  -
  -
  Just for the example. If it was always true, we wouldnât have to encode it.
  -
  -
  -Reply
  -
  -
-
-
-
-neubert says: -
-
- December 27, 2021 at 12:54 am
-
-
âByte 27: 1110 10 0 1. Length 4. Whenever we read a length, we read a distance. The distance is a range, 7-8. The extra bit we read is 0b0=0, plus 7 is Distance 7. So we look back 7 bytes and copy 4. The new output is: baabbbabaabâ
-
I think there should be an extra a before the baabbbabaab
-
-
-Reply
-
-
-
-
-inco says: -
-
- June 22, 2022 at 1:56 pm
-
-
Iâve been reading quite a bit on DEFLATE in png files and I think this is the exact level of depth i needed to crack this whole mess. Thanks so much for writing this, sources like RFC1951 talk more about abstraction and general rules but to have a few examples lined out here its EXTREMELY useful. Thanks again
-
-
-Reply
-
-
- -
  -
  -inco says: -
  -
  - June 22, 2022 at 3:38 pm
  -
  -
  Small question about reading bits: for Huffman codes of the code itself defines a length range and then the extra bits are reversed so that they can be interpreted. But should I reverse the distance or the distance extra bits as well?
  -
  -
  -Reply
  -
  -
-
-
-
-Ricardo says: -
-
- May 31, 2024 at 4:32 am
-
-
Byte 31: 10 111000: Literal âbâ
-Byte 31: 10 1110 00: Length 4, Distance 1. We look back 1 byte and copy 4. The new output is: bbbbb
-
How can I understand the distance 1 and copy 4 symbols?
-Should I repeat the âbâ four times?
-
What if the distance was 2 and copying 4 symbols?
-
-
-Reply
-
-
- -
  -
  -admin says: -
  -
  - May 31, 2024 at 5:23 pm
  -
  -
  Distance is how far back you start. Copy is how many symbols you copy.
  -You just blindly copy characters starting DISTANCE back, but since youâre appending to the string, you never run out of symbols to copy.
  -
  If your string so far was âWow, copying is really neatâ, with distance 3 and copy 10 you would add: eateateate
  -
  -
  -Reply
  -
  -
-

When you click an email address, it automatically opens in your email client. But I donât have an email client, I use webmail. I wrote a custom handler for Linux.

First write a program to open mailto links. Mailto links look like âmailto:me@mail.comâ or maybe even âmailto:me@mail.com?subject=mysubject&body=mybodyâ. Test it by hand on a few links. Mine (mailto-opener) composes a new message using my webmail.

Next, write a desktop file for the opener. Hereâs one:

#/usr/local/share/applications/mailto-opener.desktop 
-[Desktop Entry]
-Type=Application
-Name=mailto: link opener (github.com/za3k/short-programs)
-
-# The executable of the application, possibly with arguments.
-Exec=/home/zachary/.projects/short-programs/mailto-opener %u

Note the %u in the Exec= line. Thatâs required.

Now update your system mimetype database. On my Arch Linux install, I run

xdg-mime default mailto-opener.desktop x-scheme-handler/mailto

Finally, restart your browser. Really. Firefox and Chromium/Chrome both cache mimetype openers.

A related opener I added recently was for magnet links, such as are popularly used for bittorrent.

~ $ cat /usr/local/share/applications/transmission-remote.desktop 
-[Desktop Entry]
-Type=Application
-Name=transmission-remote magnet link opener
-Exec=transmission-remote <TRANSMISSION INSTANCE> -a

transmission-remote is the name of a command-line Linux program. It connects to an instance of Tranmission (a popular torrent client) running on another machine.

Year 0 â I filled 10 32-GB Kingston flash drives with random data.
Year 1 â Tested drive 1, zero bit rot. Re-wrote the drive with the same data.
Year 2 â Re-tested drive 1, zero bit rot. Tested drive 2, zero bit rot. Re-wrote both with the same data.

They have been stored in a box on my shelf, with a 1-month period in a moving van (probably below freezing) this year.

Will report back in 1 more year when I test the third ð

FAQs:

Q: Why didnât you test more kinds of drives?
A: Because I donât have unlimited energy, time and money :). I encourage you to!
Q: You know you powered the drive by reading it, right?
A: Yes, thatâs why I wrote 10 drives to begin with. We want to see how something works if left unpowered for 1 year, 2 years, etc.
Q: What drive model is this?
A: The drive tested was âKingston Digital DataTraveler SE9 32GB USB 2.0 Flash Drive (DTSE9H/32GBZ)â from Amazon, model DTSE9H/32GBZ, barcode 740617206432, WO# 8463411X001, ID 2364, bl 1933, serial id 206432TWUS008463411X001005. It was not used for anything previouslyâI bought it just for this test.
Q: Which flash type is this model?
A: We donât know. If you do know, please tell me.
Q: What data are you testing with?
A: (Repeatable) randomly generated bits
Q: What filesystem are you using? / Doesnât the filesystem do error correction?
A: Iâm writing data directly to the drive using Linuxâs block devices.

Iâve had pretty irregular sleep schedules at times, so I have some tricks for making it more regular, or moving it back/forwards. Take everything here with a spoonful of salt. All of these tricks are relatively long term (1-4 weeks) and wonât instantly fix your schedule. Most of them are from experience, with some knowledge backing them.

Also, as a note, I wake up whenever I feel like it (I donât have a day job). I have used many of these same tricks with an alarm and a day job when I had those, but I might be forgetting some details.

Quality of sleep. First off, make sure the sleep you are getting, is good. I recommend something like a Zeo ideally, because itâs hard to get a subjective feel for how well youâre sleeping. Ultimately, itâs important to you to sleep enough and sleep well. Sleeping at the right times is important to other people.

Quantity of sleep. Get enough sleep. Enough said. If you have a good quality of sleep, you donât use an alarm, and youâre waking up relaxed, youâre probably fine.

Here are some things Iâve found screw up my sleep schedule and affect my quality of sleep.

Caffein affects schedule AND quality. Caffein at 2pm, affected my quality of sleep at 2am. This is something I just COULD NOT have figured out without a Zeo. Quality of sleep is hard to diagnose.
Bright/blue light late affects schedule. Use f.lux or a similar program for your computer. Be aware that most programs of this kind donât actually WORK for your phoneâI donât use a smartphone, personally. Donât turn on room lights late at night. I find Iâm good if I turn lights off about 3 hours before I want to sleep. Turning on lights very late at night (when youâd usually be asleep), even briefly, screws up your circadian rhythm.
Light pollution affects quality. Light while you sleep sucks. I sleep next to a big window, and I often get poor sleep based on whether neighbors have their lights on. Or sometimes, I just need to sleep during the day.
- A sleep mask gives you EXCELLENT quality of sleep, but can screw up your schedule because you donât get early-morning lightâyouâll sleep longer and drift forward.
- Blackout curtains are like a sleep mask, but worse, because they donât block light as well and theyâre expensive. They could be better if you have light pollution from one window only, and theyâre okay in combination with a timer light (see below).
- Cover any electronics with lights, especially blinking or blue LEDs. I use black electrical tape.
Allergens affect quality
- Air quality massively affected my sleep. Iâd wake up with my throat scratchy, but it took a while to figure out it was affecting my sleep. I now use an ionizing air filter. The trick to air filters is that you have to regularly (once a month) clean the prefilter, and replace the main filter every 6-12 months.
- Itching. Sometimes this was just mold, which other than an air filter thereâs not much I can do about, but also make sure to regularly wash your sheets. Food (donât eat in bed!) or dust mites can make me itchy.
Other drugs may affect schedule and quality. When I started on marijuana I found it massively screwed up my sleep schedule. YMMV. Some foods can too, especially before bed.
Relaxation level affects quality. If youâre tense (neurotic especially), youâll sleep poorly. I havenât done a lot of experimentation with this one, because it comes up rarely for me. Deliberate relaxation and self-love (the hippie kind, not the sexual kind) before bed can give nice dreams, though.
Exercise before bed, or working right before bed, affects at least schedule. Both tend to keep me up.
Playing videos before bed affects quality. I might RIGHT before, like 2 minutesâI have some maybe bad habits as a bachelor. I think this doesnât let your brain relax properly, you need more âdownâ time.
Working in bed may affect schedule. As a general tip, it may be better to avoid working or otherwise being in bed during the day, to cue your body that bed=sleep.
Nightmares affects quality. Unfortunately, I canât be much help on this one. I rarely remember my dreams.
Depression affects schedule and possibly quality. Depression makes you sleep more, mania makes you sleep less. If like me you become depressed when you donât get enough sunlight, you can end up stuck nocturnal. A bright artificial light during the day is a partial solution.
Having a regular schedule is self-reinforcing. If you regularly go to bed at the same time or wake up at the same time, youâll keep doing it. Also, youâll get cranky if you donât. [A similar principle applies to dietingâif you eat meals at the same time each day, youâll get a sudden appetite then. If you donât eat meals regularly, you wonât have an appetite, or will have one only when actually hungry. But for sleep, regular is good]
Age. At 20, I needed 12 hours of sleep a day. At 30, I need only 8-10. This varies a LOT per-person, too. Some people just need more/less sleep.

If you want to move your sleep schedule forward, itâs fairly easy. Just stay up later. I have only performed the âroll forward until youâre the right timeâ operation once, and donât recommend it. Normally I hit a wall at dawn. Go forward by no more than 1 hour a day, preferably half that, or it wonât stick. If you do it for more than a few days, youâll feel weird and sleep deprived.

If you want to move your sleep schedule back a significant amount (more than just undoing a recent 1-hour forward shift) I recommend:

Do it gradually. Half an hour a day, probably more like 15 minutes. Donât bother trying to schedule it.
Have caffein AS SOON as you get up (within 15-30 minutes, the sooner the better). This moves your circadian rhythm back, and also stops you falling back asleep. Again I donât use alarms these days, but itâs a great combo to set a schedule.
You can try adjusting it by taking small (0.5mg) melatonin supplements before your usual bedtime, if youâve just drifted forward a bit
Make sure you are getting natural light if possible. If you arenât, or if itâs winter and you want extra help: hook up your lights, especially a sun lamp, to an automated timer so you get bright white light in your room around when youâd like to wake up. This can fix problems caused by blackout curtains.

Finally, Iâll leave you with a horrifying trick I learned while sleep-deprived at my first job after college. To get up while incredibly sleep deprived, set two alarms, about 30 minutes apart. After the first one, hit the alarm, chug significant portions of an energy drink on reflex while mostly asleep, then immediately fall back sleep. On the second one, actually wake upâthe caffein will help keep you awake.

I made a website to give you countdowns and let you know whether an episode is out: http://isrickandmortyout.com/. Been up since before season 4, but I forgot to post it until now.

I wanted a partition to divide my room, and I had a whiteboard sitting around. I sawed it into three parts, and connected them with hinges:

folding whiteboard

hinge (front)

hinge (back)

Iâm a little embarrassed at having done all this, since it was obvious as soon as I started the partition was way too short to work. I figured Iâd still get some experience woodworking (this is my first project). Hereâs where it went:

and I never saw it again

Action Potato

Itâs whooshing because itâs going as fast as a WEAK SPEEDBOAT.

In space no one car hear âwhooshâ sounds but action potato is so cool you still can mostly.

Last postÂ I discussed the publishing website Iâm working on.

Today I added credit card processing and address formsâitâs functionally complete and available at https://publishing.za3k.com

Next up I have to clean the site up, because it looks like this:

Iâll also add HTTPS.

Compared with last update, Â the Dead Tree Publishing website is looking nicer.

Looking better

Itâs served over HTTPS now (not needed for security, but it puts people at ease and enabled Chromeâs autocomplete) and you can order multiple books at a time.

Other than some more visual improvements, the main thing missing is proper detection of page size â my server doesnât understand about page margins, so it things books are bigger than they really are.

I consider Dead Tree Publishing to be Good Enough at this point. Itâs launched.

New site style, FAQ page

Iâm going to add support for URLs instead of uploading PDFs, and fix some bugs here for there, but itâs essentially done.

Meanwhile, Iâve already received my first physical book Iâm publishing through the site. I got this nice email from Eric Eve:

Dear Zachary,
-
-Thank you for your interest in my work. Yes, do please feel free to offer print copies of the TADS 3 Tour Guide (or any other of my TADS 3 documentation) along the lines you suggest. The only copyright right Iâm interested in enforcing is my right to be identified as the author of the work, and I'm sure that's not as issue here.
-
-Best wishes,
-
-Eric

-
-
-TheTechRobo says: -
-
- April 23, 2022 at 5:46 pm
-
-
Out of curiosity, why did you stop the service?
-
-
-Reply
-
-
- -
  -
  -admin says: -
  -
  - April 24, 2022 at 9:39 am
  -
  -
  This was launched in 2015, and youâre the first person to ask me about the the service since. That should give you an idea of the popularity. Iâm very bad at marketing, to be fair, so itâs not totally clear that means the service was undesirable.
  -
  So when I eventually ran into some problem with active maintenance (the site needed updating to keep working, or something of that kind), I just didnât.
  -
  -
  -Reply
  -
  -
-

I started work on my publishing website again (Dead Tree Publishing). The idea is to make a really, really convenient way to get a physical copy of a PDF/epub book. Think: âsend me a printed copy of this mailing list / tumblrâ. Right now things are looking encouraging.

I use a âback endâ publisher who does all the actual printing, and the one I was using before charged quite a lot and wasnât amazingly fast; I just used them because they were the only publisher who was at all up to date. Seriously, order of $100 â $200 for a 100 page book, just absolutely ridiculous levels of expensive.Â Iâm switching over to a new publisher who can offer that same book for something like $7 (maybe $12 in hardback), which is absolutely reasonable, and with similar 2-week turnaround times.

First you upload a PDF:

Uploading a book

Then I tell you what your ordering options are (hardcover, softcover, color), and what they cost. Iâm also supposed to ask you your address to ship the book, and for you to pay for it, Â but those arenât done yet.

Book-buying options

Hopefully in the next day or two Iâll have something up and running so people can order books, and then make it gradually nicer! Iâm very excited about this website existing.

Most of the following taken from :Â http://www.msfn.org/board/topic/151992-install-xp-from-usb-without-extra-tools/, just modified to include syslinux support.

Let me know if there are any omissions; it an XP installer bluescreens on boot for me so I canât actually test.

Obtain an XP iso file
Format drive with one FAT parition, marked bootable.
-
```
syslinux -i /dev/sdXX
```
-

$ cp /usr/lib/syslinux/bios/mbr.bin >/dev/sdX

-
```
$ mount /dev/sdXX /mnt
```
-

mkdir /tmp/xp_iso
-mount xp.iso /tmp/xp_iso
-cp -ar /tmp/xp_iso/* /mnt
-umount /tmp/xp_iso
-rmdir xp_iso

cp /usr/lib/syslinux/bios/{chain.c32,libutil.c32,menu.c32,libcom.c32} /mnt

cp /mnt/I386/{NTDETECT.COM,SETUPLDR.BIN,TXTSETUP.SIF} /mnt

Edit /mnt/syslinux.cfg: -

UI menu.c32# Windows XP
-LABEL windows_xp
-MENU LABEL Run Windows ^XP Setup
-COM32 chain.c32
-APPEND fs ntldr=SETUPLDR.BIN

-
```
umount /mnt
```
-
Boot from the USB stick

Sep, Oct, Nov 2014: Vietnam.

A year ago, I left my job at Streak and moved to Vietnam. I felt like I needed change. Vietnam ended up being wonderful; I was really glad I travelled with my friends Richard and Kathy, which ended up making the experience a hundred times better than it would have been otherwise. The basic environment was: everything is cheap, I newly have endless free time, I was automatically prompted by my friends in the evenings and sometimes during the day to go on small novel adventures involving physical activity, and I had little internet access. This is probably my perfect environment, and I was functioning very well (the vietnamese diet also has small, well-balanced meals which might have helped). For some reason, I was also able to intensely single-task. [Iâd like to write more about what Vietnam is like, but this article is quite long enough as it is]

While I was in Vietnam, I made a to-do list. The to-do list had all the burning projects I actually wanted to do. Iâve ended up accomplishing most of them, at a rate of one every week or two, and itâs a decent summary of what Iâve been doing since. Two things made the to-do list a success. First, it had BIG tasks. These are projects like my recent âset up an IRC serverâ or âstart a publishing companyâ. Because of that, I donât get bogged down in minutae, and the tasks are always motivating. I find I function better when I try to carefully plan around having any logistics. The second reason, which I realized today, is that I was very careful to only include tasks I was planning to do (subtly different than tasks I wanted to do). The list was descriptive, not normative, although it certainly included some things like doing taxes I wasnât wild about.

Looking at my journal and it really only starts up again in March, so Iâm going to organize this post in terms of the to-do list. There are a couple items that donât fit:

I started dating my wonderful pet, Lealend, while I was in Vietnam. I went to visit them for a month in Puerto Rico where they live. This is very very important to me (the most important thing that happened in the last year), but I donât usually write about things that personal on my blog so Iâm not doing to say much. Iâve been emotionally maturing a lot by being with them.
Conventions. I went to DEF CON, which was probably the best single week this year so far. Iâm definitely going again next year. I attended a mirixÂ [paper] in the South Bay, which ended up being stressful for transportation-related reasons but really good while I was there.Â Iâm planning on going to Burning Man this year as well.
I started contracting, thatâs how Iâve been alive for a year. Iâve been doing some work for ZincÂ and Paul ChristianoÂ on a workflowy clone, mostly. I work two hours a day average.
Iâve been developing a minecraft modpackÂ [Iâll write more about this when itâs stable], and recently taken an interest in livestreaming.

Now on to the to-do list.

-
Project: Printserver
-Success: Success but obsolete
-Description: I set up a printserver. Itâs a little raspberry pi that talks to my printer, because getting printers set up is a pain and I donât want to do it all the time. It went great, it saved me a ton of hassle to have it automatically print out my daily agenda every morning, and to just be able to transfer documents over with âscpâ.
-Future plans: Unfortunately, my printer died andÂ we only recently got a new one. I need to set it up with the new server. I could also make printing completelyÂ automatic when new files show up with scp (right now itâs manual so I can switch out paper, but my roommates would be happier with scp I think).
-
-
Project: Set up my phone so dropping/losing it isnât horrible
-Success: Partial success
-Description: I wanted to root and thenÂ automatically backup my phone. I did figure out how to do as much backing up as I can, and it is automatic. Unfortunately it turns out most of the filesystem (including SMS) just isnât available over Media Transfer ProtocolÂ which android uses to display files, so I had to special case the things I desperately needed backed up. Iâd prefer the state of the world let me back up everything on the phone, but thatâs as much work as Iâm willing to do.
-
-
-
Project: Get digital copies of allÂ books I own
-Success: Success
-Description: I got digital copies of all books I own via a combination of pirating, buying copies, and getting the books scanned by a service. I did not get rid of the physical books.
-
-
-
Project: Switch to private email
-Success: Not done
-Description: I get a little nervous entrusting Google (or any third party) with the ability to read, lose, or add restrictions on what I can do with my email. I want to set up my own email address (za3k@za3k.com) and have it be my main point of contact. My email does work, but I canât send outgoing email, and I havenât switched everything over to it for that reason.
-
-
Project: Download ArXiV
-Success: Done
-Description: As an archive nut, I worry that the ArXiVÂ collection, one of the nicer collections of scientific papers I access regularly, might someday go down or get censored. I downloaded a copy and stashed it away somewhere. Unfortunately ArXiVâs licenses they get papers under doesnât permit redistribution, so I canât publicly host it. (This was really cool but I had to decide whether I was going toÂ publicly mention, since itâs a legal gray area)
-Future plans: Someone (not me) should host a torrent. Contact me and I can get you a copy.
-
-
Project: Pack and unpack storage bins (trip to vietnam)
-Success: Success
-Description: Okay I know this sounds stupid, but I spent about a month packing up to go to Vietnam, and all my physical stuff has stayed organized ever since. Thatâs a really big change for me.
-
-
Project: Host an IRC server
-Success: Success
-
-
-
Project: Make hibernate work on my laptop
-Success:Â Success
-Description: This involved switching partitioning aroundÂ since btrfs doesnât support swap files. If I recall, my setup is now a swap partition and a root btrfs partition, inside LVM, inside LUKS.
-
-
-
Project: Extract bitcoins
-Success: Success
-Description: Extract bitcoins from all my computers and centralize them in one place
-
-
Project: iPhone
-Success: Success
-Description: Back up all my personal data from my iPhone, clear the contents, and sell it.
-
-
-
Project: N-grams
-Success: Obsolete
-Description: The Google N-grams dataset from their book scanning project is freely available, but in a terrible format (split across set-size file chunks, but in random rather than sorted order). My plan was to convert the formatting and offer it as a torrent / s3 bucket. Google has corrected the problem in a revised version of theÂ dataset.
-
-
-
Project: NNTP over tor
-Success: Didnât do
-Description: I run a private newsserver, and I wanted to let people access the newsserver (and anything else on that physical server) over tor. I decided the newsserver was too dead to bother with, and I didnât feel enthusiastic about setting up tor, so I dropped the project.
-Future plans: I donât care about the original project, but if thereâs a compelling stimulus, IÂ want to set up tor for my server to learn how and leave flexibility.
-
-
-
Project: Textmode backup
-Success: Success
-Description: âtextmodeâ is the name of a virtual machine on my OS X machine. The project was to back up contents of the machine once, and then delete the virtual machine
-
-
-
Project: Post pdfmailer website
-Success: Success
-Description: I wanted people to be able to get a physical copy of a pdf document they had mailed to them. I think this project was an especial success, because Iâd been failing at an over-engineered version of this off and on for a year. I decided to have the website email me instead of trying to do everything automatically, and ended up getting the books to be a factor of 10 cheaper or so by going with a publisher with no API.
-Future plans:Â Iâd like to popularize the website more. I think there are also some small technical improvements to be made. Iâm not going to automate things unless it starts using up a lot of my time to process requests myself.
-
[Censored project involving an arbitrage opportunity I havenât cornered]
Project: Back up email
-Success: Success
-
Project: Flatten backups
-Success: Good enough
-Description:Â Oh just go read the XKCD. Now imagine youâve beenÂ archiving computers onto other computers for 15 years, and buy cheap laptops.
-
Project: QR codes for ebooks
-Success: Success
-
-
Project: Business cards
-Success: Not done
-Description: Make some personal business cards
-
-
Project: QR Punchcodes
-Success: Didnât do
-Description: So you know how QR codes can contain any data? That means you could show them to a camera and the camera could run any code. Like, code to wait for another couple of QR codes, or to print out some more QR codesâ¦
-
-
[Censored project involving an arbitrage opportunity I havenât cornered]
-
-
[Censored project involving a mildly illegal thing]
-
-
Project: Make a desk out of cardboard
-Success: Ongoing
-Description: I want to make a desk out of cardboard, because it sounds fun. Iâm proud of doing the design right here. Iâve finished mocking it out of cardstock, and actually noticed a lot of flaws and fixed the design instead of hoping them away. Now I mostly have to get the cardboard and make it, should be fun.
-
-
-
Project: Make a whiteboard partition
-Success: Success
-
-
Project: Write about paper backups
-Success: Success
-
-
-
Project: Sort physical scans
-Success: Success
-Description: As part of packing up all my possessions to go to Vietnam, I scanned every physical document I own (and mostly threw them out). Twenty years of stuff is a lot of stuff, but I eventually sorted it all out. Iâve been increasingly finding that a flat folder structure ends up working out best for me in the long term, so thatâs what I used.
-
-
-
Project: Two-location backup
-Success: Not done
-Description: My backup server uses RAID-1, but Iâd like to have a second copy on an external hard drive somewhere. Â Also, I have a bunch of external hard drives which are currently not backed up anywhere (mostly with stuff like movies) which Iâd like toÂ have some kind of redundancy
-
-
Project: Treemap finances
-Success: Not done
-Description: Iâve made my finances public, but my analysis tools arenât great. Iâd like to update some old work Iâve done and add a web interface to see where I spent money during a particular time span, using a treemap display.
-
-
Project: Archive Github (aka download all the code in the world)
-Success: Not done, on hold
-Description: Iâm kind of burnt out on archiving tasks lately, so this doesnât sound fun to me. I decided to work with archive team on this one. Itâll get done if it sounds low-stress and no one else seems to be doing it, but itâs less likely than the older archiving projects, despite being important for the world.
-
-
Project: Encrypt backup
-Success: Not done
-Description: Iâd like a way to back up my data to untrusted media, like tarsnap does, especially a way that avoids leaking file metadata (like access times and file lengths). Failing that, I should at least encrypt the drive backups are to so I can turn off that computer if needed.
-
-
Project: Gwernify
-Success: Not done
-Description: Gwern writes about how to protect links against link rot. He does this for all links on his website. I ambitiously plan to automatically save a copy of every site I visit (not just the actual URL I visit ideally, but the whole page).
-

I just wrote the first pass at youtube-autodl, a tool for automatically downloading youtube videos. Itâs inspired by Popcorn Time, a similar program I never ended up using, for automatically pirating the latest video from a TV series coming out.

You explain what you want to download, where you want to download it to, and how to name videoes. youtube-autodl takes care of the rest, including de-duplication and downloading things ones.

The easiest way to understand it is to take a look at the example config file, which is my actual config file.

Personally, I find youtube is pushing âwatch this relatedâ video and main-page feeds more and more, to the point where they actually succeed with me. I donât want to accidentally waste time, so I wanted a way to avoid visiting youtube.com. This is my solution.

The Zorchpad needs a custom keyboard. Its power budget is only 1mW, and thereâs just nothing available in that range. So, I need to make a custom keyboard. I started reading up on how to make your ownâespecially the electronics.

I donât know how to make a PCB:

PCB from HacKeyboard

Or how to attach headers to the inside of an existing keyboard, which looks like thisâ:

Mapping the Innards of a Keyboard : 7 Steps (with Pictures) - Instructables

But I found a project called GOLEM with an excellent guide to making your own keyboard. Here is their wiring:

GOLEM Macropad

I can do that! They got me out of a major rut.

Their advice walks you through how to do a small keyboard in a cardboard plate. I did a few keys, gauged the effort, and decided to use my 3D printer. Cutting out 50-60 keys precisely by hand doesnât sound easy. Worse, if you mess up, you have to start over. In plastic, I canât mess up halfway, and the spacers to support the keyboard can be part of the print.

Above, Iâm designing a âsamplerâ keyboard in CAD (OpenSCAD). I want to iron out problems in my process before I try a full-size keyboard. Below, Prusa-Slic3r is slicing the finished model for my 3D printer to print.

Hereâs the finished sampler keyboard:

Currently Iâm waiting on keycaps and switches ordered from China, and then Iâll put together my finished keyboard. But I have been making some progress in the meantime. Hereâs the layout Iâm going to try.

And Iâve started streaming some development of a case and keyboard on Twitch (Tue/Thu 12pm noon, EDT). Feel free to join! Anyone can watch, but you need an account to chat.

Iâve gotten to the point in Zorchpad development where Iâd like to see how the whole thing fits together and if there will be any insurmountable problems. Weâre still trying to figure out some things likeâwill it have one screen or two? What form factor will it be? Will the keyboard fold in half? So I put together a cardboard model.

This model has:

A power switch. Iâm assuming the very first prototype will run on battery, not solar like the final one.
Two memory-in-pixel screens (total power usage: 0.001 W)
One e-ink display (total power usage: variable/unknown)
An apollo3 artemis board, which includes RAM, CPU, microphone, and BTLE (not pictured, total power usage about 0.0005 W)
One flash chip (not pictured, power usage variable)
A battery slot. Iâm using AAA for my convenience (Holds: 3000 joules = ~20 days power)
An audio jack for headphones
A microSD slot
A custom keyboard (total power usage: variable/unknown)
The keyboard is closely modeled off a standard one, for now.

Immediately, a few problems pop out:

Itâs fairly long. This will stick out of my pocket. This is with very closely spaced keys on a somewhat reduced keyboard.
Thereâs not really a great place to put solar panels. Itâs has almost zero free space, plus thereâs going to be a lot of wiring. Are we going to drill holes through the solar panel to let wires pass through? Also, I really havenât been able to figure out how many cm² of solar we need.
Itâs hard to get the screen to stay propped up on my cardboard model. Iâd like a solution that doesnât use hinges, since those tend to loosen over time.

My next step will probably be to make a custom working keyboard. Then, Iâll make an entire working zorchpad. Both will be either cardboard or 3d-printed (whichever is easier).

Iâve been designing a keyboard and case for the zorchpad.

There are four pieces in the first iteration.

A top bottom base, to enclose the keyboard electronics.

A keyboard plate. The keys fit into the holes here. You type on the top, electronics go in the bottom.

A top plate. You see the screens, and switches through the cutouts. Otherwise, it keeps the behind-the-scenes wiring out of sight.

And finally, the top piece.

Here are the pieces in correct position. In the top will be the screens and battery. The bottom is a keyboard you type on. The whole things is meant to fold on a hinge, much like a laptop.

The same pieces, spread out.

There were many, many problems with the first design and the first print. Iâll talk about them (and my fixes) in my next post.

Another update on the zorchpad. We now have a working 16-button keyboard (sadly no QWERTY yet). Here you can see a simple typing program that shows what you type on screen.

As mentioned in a previous post, the reason weâre using a custom keyboard is to stay to low power usageâmuch lower than a standard keyboard.

So far we have:

Screen display â both pixels and letters
Sound output (simple beeps for now)
Keyboard input

-#include - -int main(int argc, char **argv) { - initscr(); - - refresh(); - printw("Hello world"); - getch(); - - endwin(); - return 0; -} -``` - -Aha, so itâs something in ncurses, maybe. Letâs dive deeper. - -So *initscr()* presumably saves the state in some fashion. *endwin()* definitely restores it, because if we comment that out, the terminal stops being restored. Since *initscr()* probably does lots of other irrelevant logic, we could take a look at *endwin()* to dive in. But letâs do something even simpler first. - -As background, the linux command line is pretending to be an obsolete piece of hardware called a [terminal](https://en.wikipedia.org/wiki/Computer_terminal). Specifically, itâs pretending to be a model called the DEC VT100 (or a later one, but theyâre mostly backwards compatible). The terminal accepted text over a wire, and printed it to the screen. - -When it got special text, it would do special things. These are captured today as âescape codesââspecial non-printable characters, which cause your software terminal to also do special things. What kind of special things? Well one example escape code is âBackspaceâ, which deletes the last character. Another code is â\\r\\nâ (carriage return; new line), which goes to the beginning of the line, and down one. - -I suspect the answer to whatâs happening with save and restore of my terminal might be some magic escape codes. So letâs just redirect the output to a file, and see what data is being center over the virtual wire to our virtual terminal. - -``` -$ ./editor >magic.txt -$ xxd < magic.txt -00000000: 1b28 421b 2930 1b5b 3f31 3034 3968 1b5b .(B.)0.[?1049h.[ -00000010: 313b 3530 721b 5b6d 0f1b 5b34 6c1b 5b48 1;50r.[m..[4l.[H -00000020: 1b5b 4a48 656c 6c6f 2077 6f72 6c64 6a1b .[JHello worldj. -00000030: 5b35 303b 3148 1b5b 3f31 3034 396c 0d1b [50;1H.[?1049l.. -00000040: 5b3f 316c 1b3e [?1l.> -``` - -Well, thatâs certainly a bunch of magic. Now something cool happens: - -``` -$ cat magic.txt -``` - -This command does *nothing visible*. It doesnât print âHello worldâ, even though thatâs in the file. In other words, itâs printing Hello world, then really quick resetting the terminal. Just too fast for my poor human eyes to see. - -Weâve confirmed the escape code theory! This file has everything we need. We can look at the source code to ncurses if weâre curious, but we donât need to (and I wonât). - -One thing I immediately see in this file, is that it *doesnât* seem to contain the words that were on screen. So itâs not that the program read what was on screen, and printed it back later. Rather, there are some magic escape sequences happening to save and restore the terminal. - -Okay, so somewhere in those 70 bytes is a magic code or two we want. Letâs examine all the bytes. - -What kinds of escape codes appear here? Hex **0x1b** is ESC, basically *the* escape sequenceâit cues the terminal in that a special escape code is coming. **0x1b9b** ( **ESC** followed by **\[** )is the CSI escape code. [DEC private codes](https://en.wikipedia.org/wiki/VT100) refer to other escape sequences used by the DEC terminals like the VT00 (Iâll just shorten this to âDECâ below). - -Without further ado, letâs break down those 70 bytes. Apologies for any errors belowâcorrect me in the comments. - -- **0x1b** (B - - Set first character set to US ASCII \[[xterm DEC guide](https://www.xfree86.org/current/ctlseqs.html)\] -- **0x1b** )0 - - Set second character set to âDEC Special Character and Line Drawing Setâ \[[xterm DEC guide](https://www.xfree86.org/current/ctlseqs.html)\] -- **0x1b9b** ?1049h - - **Save cursor as in DECSC and use Alternate Screen Buffer, clearing it first.** \[[xterm CSI guide](https://www.xfree86.org/current/ctlseqs.html)\] -- ****0x1b**9b** 1;49r - - DECSTBM: Set scrolling region to rows 1-49 \[[xterm CSI guide](https://www.xfree86.org/current/ctlseqs.html)\] - (When I ran the program, my terminal was 49 lines tall inside tmuxâso the whole terminal in other words.) -- **0x1b9b** m - - (Empty) color and style set comand \[[ANSI/CSI](https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797)\] - I think this could be left out entirely. -- **0x0f (Ctrl-O)** - - Shift In. Use the standard character set. \[[xterm](https://www.xfree86.org/current/ctlseqs.html)\] -- **0x1b9b** 4l - - RM: Replace Mode / IRM \[[xterm CSI guide](https://www.xfree86.org/current/ctlseqs.html)\] - Typing doesnât shift everything over, it writes over existing content. -- **0x1b9b** H - - Move the cursor to âhomeâ (top left of the screen) \[[ANSI/CSI](https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797)\] -- **0x1b9b** J - - Clear the screen \[[ANSI/CSI](https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797)\] -- Hello worldj - - The program prints âHello worldâ. The final j was echoed when I pressed âjâ to exit. -- **0x1b9b** 49;1H - - Move the cursor to line 49, column 1 â the bottom-left corner \[[ANSI/CSI](https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797)\] -- **0x1b9b** ?1049l - - **Use Normal Screen Buffer and restore cursor as in DECRC** \[[xterm CSI guide](https://www.xfree86.org/current/ctlseqs.html)\] -- **0x0d (\\r or Ctrl-M)** - - Carriage return \[[xterm](https://www.xfree86.org/current/ctlseqs.html)\] - Moves cursor to column 0 -- ****0x1b**9b** ?1l - - DECCKM reset: Re-enables the cursor? \[[VT100](https://vt100.net/docs/vt220-rm/chapter4.html#S4.6.18)\] -- **0x1b** \> - - DECONM: Normal Keypad \[[xterm DEC guide](https://www.xfree86.org/current/ctlseqs.html), [VT100](https://vt100.net/docs/vt220-rm/chapter4.html#S4.6.18)\] - -OK, solved. The magic save bytes are **1b 9b 3f 31 30 34 39 68** ( \[?1049h). The magic restore bytes are **1b 9b 3f 31 30 34 39 6c** ( \[?1049l). And xterm or tmux is doing the save/restore magic, based on seeing this escape mode. - -Hmm, how minimal can we get a working file, I wonder? - -``` -#!/bin/sh -echo -ne '\x1b[?1049h' # Save terminal -echo -ne '\x1b[H' # Home the cursor -echo "Hello world" -sleep 1 -echo -ne '\x1b[?1049l' # Restore terminal -``` - -Yep. That works fine. - -ANSI: [https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797](https://gist.github.com/fnky/458719343aabd01cfb17a3a4f7296797) -DEC: [https://vt100.net/emu/ctrlseq\_dec.html](https://vt100.net/emu/ctrlseq_dec.html) -DEC: [https://vt100.net/docs/vt220-rm/chapter4.html#S4.6.18](https://vt100.net/docs/vt220-rm/chapter4.html#S4.6.18) -xtermâs control sequences: [https://www.xfree86.org/current/ctlseqs.html](https://www.xfree86.org/current/ctlseqs.html) diff --git a/posts-md/archiving-twitter.md b/posts-md/archiving-twitter.md deleted file mode 100644 index 5b78a15..0000000 --- a/posts-md/archiving-twitter.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -author: admin -categories: -- Technical -date: 2014-11-23 14:35:14-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- apis -- backup -- jq -- system administration -- twitter -title: Archiving twitter -updated: 2014-11-24 13:12:44-07:00 -wordpress_id: 61 -wordpress_slug: archiving-twitter ---- -([Output](https://za3k.com/~twitter_archive/)) - -I wanted to archive twitter so that I could - -1. Make sure old content was easily available -2. Read twitter in a one-per-line format without ever logging into the site - -[twitter\_ebooks](https://github.com/mispy/twitter_ebooks)Â is a frameworkÂ to make twitter bots, but it includes anÂ âarchiveâ componentÂ to fetch historical account content which is apparently unique in that it 1) works with current TLS and 2) works the current twitter API. It stores the tweets in a JSON format which presumably matches the API return values. Usage is simple: - -while read account -do - [ebooks](https://github.com/mispy/twitter_ebooks) archive "${account}" "archive/${account}.json" - [jq](http://stedolan.github.io/jq/) -r 'reverse | .\[\] | "\\(.created\_at|@sh)\\t \\(.text|@sh)"' "archive/${account}.json" >"archive/${account}.txt" -done - -[![](https://blog.za3k.com/wp-content/uploads/2023/06/aluminium_parts-150x150.jpg)](https://blog.za3k.com/wp-content/uploads/2023/06/aluminium_parts-scaled.jpg) - -[![](https://blog.za3k.com/wp-content/uploads/2023/06/aluminium_layers-150x150.jpg)](https://blog.za3k.com/wp-content/uploads/2023/06/aluminium_layers-scaled.jpg) - -[![](https://blog.za3k.com/wp-content/uploads/2023/06/aluminium_03-150x150.jpg)](https://blog.za3k.com/wp-content/uploads/2023/06/aluminium_03-scaled.jpg) - -[![](https://blog.za3k.com/wp-content/uploads/2023/06/aluminium_01-150x150.jpg)](https://blog.za3k.com/wp-content/uploads/2023/06/aluminium_01-scaled.jpg) - -[![](https://blog.za3k.com/wp-content/uploads/2023/06/aluminium_02-150x150.jpg)](https://blog.za3k.com/wp-content/uploads/2023/06/aluminium_02-scaled.jpg) - -[![](https://blog.za3k.com/wp-content/uploads/2023/06/side_view-1024x576.jpg)](https://blog.za3k.com/wp-content/uploads/2023/06/side_view-scaled.jpg) - - - -And hereâs one using graphite from drawing hard with a #2 pencil.. Graphite, it turns out, works terribly, and I couldnât read a signal halfway down the index card. Despite what people have told me, Iâm not yet convinced you can make a conductive wire out of it. - -

- -[![](https://blog.za3k.com/wp-content/uploads/2023/06/graphite_parts-1024x442.jpg)](https://blog.za3k.com/wp-content/uploads/2023/06/graphite_parts-scaled.jpg) - -[![](https://blog.za3k.com/wp-content/uploads/2023/06/graphite_done-1024x666.jpg)](https://blog.za3k.com/wp-content/uploads/2023/06/graphite_done-scaled.jpg) - - diff --git a/posts-md/hack-a-day-day-06.md b/posts-md/hack-a-day-day-06.md deleted file mode 100644 index 2d7c186..0000000 --- a/posts-md/hack-a-day-day-06.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -author: admin -categories: -- Technical -date: 2023-11-07 07:33:14-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- graphics -- hack-a-day -- raytracing -title: 'Hack-A-Day, Day 06: Raytracing Redux (realtime video)' -updated: 2023-11-11 11:39:03-07:00 -wordpress_id: 1152 -wordpress_slug: hack-a-day-day-06 ---- -Todayâs update is a short one. I ported my raytracer from [day 02](https://blog.za3k.com/hack-a-day-day-2-raytracing/), to the Nvidia GPU: [ha3k-06-raytracer](https://github.com/za3k/ha3k-06-raytracer) - -The visuals are pretty much the same. Incidentally I discovered the striations on the ground disappear if we increase the floating point precision. - -

- -[![](https://blog.za3k.com/wp-content/uploads/2023/11/v15b-300x225.png)](https://blog.za3k.com/wp-content/uploads/2023/11/v15b.png) - -[![](https://blog.za3k.com/wp-content/uploads/2023/11/v16-300x225.png)](https://blog.za3k.com/wp-content/uploads/2023/11/v16.png) - - - -Render on the GPU is 30x faster (0.05 fps -> 3 fps). Thatâs still not very fast. - -I didnât get video working yesterday, or anything else visually new. I will call this one a failure overall, because I have nothing interesting to show off. I learned stuff and made progress though, so itâs not so bad. - -Hereâs a working video! - - diff --git a/posts-md/hack-a-day-hack-a-clock.md b/posts-md/hack-a-day-hack-a-clock.md deleted file mode 100644 index fb785e6..0000000 --- a/posts-md/hack-a-day-hack-a-clock.md +++ /dev/null @@ -1,24 +0,0 @@ ---- -author: admin -categories: -- Non-Technical -- Technical -date: 2022-11-25 21:08:03-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- hack-a-day -- november -- throwaway -- time -title: 'Hack-A-Day: Hack-A-Clock' -updated: 2022-11-25 21:08:03-07:00 -wordpress_id: 934 -wordpress_slug: hack-a-day-hack-a-clock ---- -Thursdayâs project was [Hack-A-Clock](https://tilde.za3k.com/hackaday/clock/) ([demo](https://tilde.za3k.com/hackaday/clock/), [source](https://github.com/za3k/day24_clock)). It is a decimal time clock, displaying the time in revolutionary french time (minus their weird calendar). - -[![caption:https://tilde.za3k.com/hackaday/clock/](https://blog.za3k.com/wp-content/uploads/2022/11/screenshot-21.png)](https://blog.za3k.com/wp-content/uploads/2022/11/screenshot-21.png) - -This is another âphone it inâ project but I think it would have been okay with more accompanying explanation and better styling. diff --git a/posts-md/hack-a-day-hack-a-hang.md b/posts-md/hack-a-day-hack-a-hang.md deleted file mode 100644 index 700dd50..0000000 --- a/posts-md/hack-a-day-hack-a-hang.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -author: admin -categories: -- Non-Technical -- Technical -date: 2022-11-19 21:04:04-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- audio -- hack-a-day -- november -- social -- throwaway -- video -- webrtc -title: 'Hack-A-Day: Hack-A-Hang' -updated: 2022-11-19 21:13:00-07:00 -wordpress_id: 918 -wordpress_slug: hack-a-day-hack-a-hang ---- -Itâs november, and Iâve decided this month that Iâm going to do 30 projects in 30 days. Itâs an all-month hack-a-thon! - -Todayâs project is [Hack-A-Hang](https://tilde.za3k.com/hackaday/hang/) ([demo](https://tilde.za3k.com/hackaday/hang/), [source](https://github.com/za3k/day19_hang)). Itâs a place to hang out. It has text chat, video, and audio. - -**Hack-A-Hang is NOT WORKING.** - -Unfortunately while everything works great on my machine, thereâs a bad problem in production, and I ran out of time on this one. Iâll try to get it fixed before the end of the month if itâs easy. - -[![](https://blog.za3k.com/wp-content/uploads/2022/11/screenshot-17.png)](https://tilde.za3k.com/hackaday/hang/) - -Hoo boy, this was one of the technically hardest ones so far. WebRTC is no joke. And not hard in a way where you have to think, hard in a way where the debugging tools are terrible. (Drag and drop was another tough one) diff --git a/posts-md/known-good/e-ink-laptop.md b/posts-md/known-good/e-ink-laptop.md deleted file mode 100644 index e996de4..0000000 --- a/posts-md/known-good/e-ink-laptop.md +++ /dev/null @@ -1,90 +0,0 @@ ---- -author: admin -categories: -- Non-Technical -- Technical -date: 2022-10-12 15:45:27-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- eink -- electronics -- physical -- prototype -title: "e-ink \u201Claptop\u201D" -updated: 2022-10-13 10:30:55-07:00 -wordpress_id: 801 -wordpress_slug: e-ink-laptop ---- -Iâve been prototyping an e-ink laptop. - -[![alt:a wooden box with a keyboard inside and an e-ink screen mounted to it](https://blog.za3k.com/wp-content/uploads/2022/10/front_view_open-1024x768.jpg)](https://blog.za3k.com/wp-content/uploads/2022/10/front_view_open-scaled.jpg) - -[![caption:Closed âlaptopâ](https://blog.za3k.com/wp-content/uploads/2022/10/front_view-300x225.jpg)](https://blog.za3k.com/wp-content/uploads/2022/10/front_view-scaled.jpg) - -Iâm not the first, there have been many other such devices before. I came up with the idea independently, but the specifics are heavily inspired by the [Ultimate Writer](https://alternativebit.fr/posts/ultimate-writer/) by NinjaTrappeur in 2018. Similar to him, my use case is typing without distractions, and reading books. E-ink displays are quite slow to update, so I donât think it can serve as a general purpose computer. Hereâs a video of it in action. It operates at one frame per second. - -

- -The electronics are not fully done. They need better secured, and Iâm going to redo the cabling and power back. - -[![caption:I broke a screen over-tightening a nut. That said, I like this look pretty well! If the lid was thicker, I know how to avoid screws on the other side, too.](https://blog.za3k.com/wp-content/uploads/2022/10/screen_closeup-1024x768.jpg)](https://blog.za3k.com/wp-content/uploads/2022/10/screen_closeup-scaled.jpg) - -[![caption:Early screen progress. I got something to display, but not what I wanted.](https://blog.za3k.com/wp-content/uploads/2022/10/early_garbage-crop-300x224.jpg)](https://blog.za3k.com/wp-content/uploads/2022/10/early_garbage-crop-scaled.jpg) - -[![caption:I found a really nice, cheap mechanical keyboard on ebay. The main downside is that itâs heavyâ730g. It also consumes heavy amounts of power, even when not in use. I have a nearly identical keyboard that doesnât, which Iâll use for v2.](https://blog.za3k.com/wp-content/uploads/2022/10/keyboard_closeup-300x225.jpg)](https://blog.za3k.com/wp-content/uploads/2022/10/keyboard_closeup-scaled.jpg) - -[![caption:I made my own lithium-ion battery pack. It works well, but it doesnât quite fit so Iâm going to redo it with one less cell. It also needs an on/off switch and a right angle USB cable.](https://blog.za3k.com/wp-content/uploads/2022/10/battery_back_closeup-300x225.jpg)](https://blog.za3k.com/wp-content/uploads/2022/10/battery_back_closeup-scaled.jpg) - -[![caption:The prototype is powered by a Raspberry Pi 3. The final version will use a microcontroller to save power. The Pi Zero can also be swapped in with no changes, and uses a third of the power. But itâs noticeably slower and takes 30 seconds to boot. For prototyping Iâm using the Pi 3 for now.](https://blog.za3k.com/wp-content/uploads/2022/10/pi_closeup-300x225.jpg)](https://blog.za3k.com/wp-content/uploads/2022/10/pi_closeup-scaled.jpg) - -Iâm not the best woodworker, but Iâm slowly learning. Here are pictures of case and lid action. - -[![caption:Hinged lid. The screen is on the bottom of the lid.](https://blog.za3k.com/wp-content/uploads/2022/10/added_back_stops-300x225.jpg)](https://blog.za3k.com/wp-content/uploads/2022/10/added_back_stops-scaled.jpg) - -[![caption:A wooden stop on each side](https://blog.za3k.com/wp-content/uploads/2022/10/back_stop-300x225.jpg)](https://blog.za3k.com/wp-content/uploads/2022/10/back_stop-scaled.jpg) - -[![caption:Wooden stop with lid open. It hits the bottom, bringing the lid/screen to a rest at vertical.](https://blog.za3k.com/wp-content/uploads/2022/10/back_stop_action-300x225.jpg)](https://blog.za3k.com/wp-content/uploads/2022/10/back_stop_action-scaled.jpg) - -[![caption:Latches on the side](https://blog.za3k.com/wp-content/uploads/2022/10/hinge-300x225.jpg)](https://blog.za3k.com/wp-content/uploads/2022/10/hinge-scaled.jpg) - -[![caption:Donât put hinges sideways into plywood. But if you do, drill big pilot holes. Out of six screw, one cracked a little.](https://blog.za3k.com/wp-content/uploads/2022/10/hinge_crack-300x225.jpg)](https://blog.za3k.com/wp-content/uploads/2022/10/hinge_crack-scaled.jpg) - -On the software end, shout outs to: - -- the creator of the [ultimate-writer](https://github.com/NinjaTrappeur/ultimate-writer) software, NinjaTrappeur, who has been encouraging (and explained the right way to rewrite the stack, if you wanted to today). -- Ben Krasnow, who made a [video](https://www.youtube.com/watch?v=MsbiO8EAsGw&ab_channel=AppliedScience) about how to hack partial refresh on an e-ink display. - -Thereâs a few things Iâd like to polish stillâeven as a prototype this isnât fully done. - -- The raspberry pi and battery pack are currently sitting loose. They need secured, especially since they can fall out the open front. -- The software has some major problems. It doesnât support Control-C, etc in linux, a must, and it doesnât update the screen at boot until you press a key, which would be nice to fix. -- Thereâs no power switch. Right now you have to unplug it manually. -- Iâd like to add a carrying handle. -- Iâd like to tuck away the electronics behind a panel. Theyâre ugly. -- The wood looks rough in a few places. I want to hide some splintered wood, screw holes, etc. -- The USB cables have too much stress on them. I need to make a little more room in the wood, and use a right-angled connector in one place. - -Thereâs also no default software, but thatâs a feature. A prototype is for figuring out how I want the interface to work, and what I want it to do. - -Parts list - -- [7.5 inch e-ink screen](https://www.waveshare.com/7.5inch-e-paper-hat.htm) from Waveshare (not particularly good) â $60 -- Raspberry Pi 3 (Pi Zero, etc also work with no changes) â $35 (but unavailable) -- microsd card â $7 -- Plywood and boards, wood glue â $15 -- [Plexiglass](https://www.amazon.com/gp/product/B088LXM1P1) (to cover screen) â $10 -- Bolts, washers, and nuts to secure it. â $5 -- Circular [window latch](https://www.amazon.com/dp/B000CSGD1U) x2 â $8 (or use $10 smaller [version](https://www.amazon.com/dp/B09ZTLLC6K)) -- Hinge x2 â $2 -- Total: $142 - -Power budget (at 5V): - -- Keyboard: 500mW. Other USB keyboards use zero to within my measurement abilities. -- Screen: 0-250mW when updating. Hard to measure. -- Pi 3: 2000mW. I have the wifi chip enabled (the default) but Iâm not actively connected to wifi. -- Pi Zero W: 650mW - -A real-life test showed 5-6 hour battery life. Theory says (13Wh/battery \* 4 batteries / 2.7 watts)=20 hours battery life. Iâm investigating the discrepancy. In theory, swapping for a Pi Zero W and a better keyboard would give 72-hour battery life. diff --git a/posts-md/migrating-an-existing-debian-installation-to-encrypted-root.md b/posts-md/migrating-an-existing-debian-installation-to-encrypted-root.md deleted file mode 100644 index 487df9b..0000000 --- a/posts-md/migrating-an-existing-debian-installation-to-encrypted-root.md +++ /dev/null @@ -1,100 +0,0 @@ ---- -author: admin -categories: -- Technical -date: 2021-06-11 13:28:52-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- debian -- linux -- system administration -title: Migrating an existing debian installation to encrypted root -updated: 2021-06-11 18:06:48-07:00 -wordpress_id: 606 -wordpress_slug: migrating-an-existing-debian-installation-to-encrypted-root ---- -In this article, I migrate an existing debian 10 buster release, from an unencrypted root drive, to an encrypted root. I used a second hard drive because itâs saferâthis is NOT an in-place migration guide. We will be encrypting / (root) only, not /boot. My computer uses UEFI. This guide **is specific to debian**âI happen to know these steps would be different on Arch Linux, for example. They probably work great on a different debian version, and might even work on something debian-based like Ubuntu. - -In [part 2](https://blog.za3k.com/encrypted-root-on-debian-part-2-unattended-boot/), I add an optional extra where root decrypts using a special USB stick rather than a keyboard passphrase, for unattended boot. - -Apologies if I forget any stepsâI wrote this after I did the migration, and not during, so itâs not copy-paste. - -Q: Why arenât we encrypting /boot too? - -1. Encrypting /boot doesnât add much security. Anyone can guess whatâs on my /bootâitâs the same as on everyone debian distro. And encrypting /boot doesnât prevent tamperingâsomeone can easily replace my encrypted partition by an unencrypted one without my noticing. Something like [Secure Boot](https://www.rodsbooks.com/efi-bootloaders/secureboot.html) would resist tampering, but still doesnât require an encrypted /boot. -2. I pull a special trick in [part 2](https://blog.za3k.com/encrypted-root-on-debian-part-2-unattended-boot/). Grub2âs has new built-in encryption support, which is what would allow encrypting /boot. But grub2 canât handle keyfiles or keyscripts as of writing, which I use. - -**How boot works** - -For anyone that doesnât know, hereâs how a typical boot process works: - -1. Your computer has built-in firmware, which on my computer meets a standard called UEFI. On older computers this is called BIOS. The firmware is built-in, closed-source, and often specific to your computer. You can replace it with something open-source if you wish. -2. The firmware has some settings for what order to boot hard disks, CD drives, and USB sticks in. The firmware tries each option in turn, failing and using the next if needed. -3. At the beginning of each hard disk is a *partition table*, a VERY short info section containing information about what partitions are on the disk, and where they are. There are two partition table types: MBR (older) and GPT (newer). UEFI can only read GPT partition tables. The first thing the firmware does for each boot disk is read the partition table, to figure out which partitions are there. -4. For UEFI, the firmware looks for an âEFIâ partition on the boot diskâa special partition which contains bootloader executables. EFI always has a FAT filesystem on it. The firmware runs an EFI executable from the partitionâwhich one is configured in the UEFI settings. In my setup thereâs only one executableâthe grub2 bootloaderâso it runs that without special configuration. -5. Grub2 starts. The first thing Grub2 does isâ¦ read the partition table(s) again. It finds the /boot partition, which contains grub.cfg, and reads grub.cfg. (There is a file in the efi partition right next to the executable, which tells grub where and how to find /boot/grub.cfg. This second file is confusingly also called grub.cfg, so letâs forget it exists, we donât care about it). -6. Grub2 invokes the Linux Kernel specified in grub.cfg, with the options specified in grub.cfg, including the an option to use a particular initramfs. Both the Linux kernel and the initramfs are also in /boot. -7. Now the kernel starts, using the initramfs. initramfs is a tiny, compressed, read-only filesystem only used in the bootloading process. The initramfsâs only job is to find the real root filesystem and open it. grub2 is pretty smart/big, which means initramfs may not have anything left to do on your system before you added encryption. **If youâre doing decryption, it happens here.** This is also how Linux handles weird filesystems (ZFS, btrfs, squashfs), some network filesystems, or hardware the bootloader doesnât know about. At the end of the process, we now have switched over to the REAL root filesystem. -8. The kernel starts. We are now big boys who can take care of ourselves, and the bootloading process is over. The kernel always runs /bin/init from the filesystem, which on my system is a symlink to systemd. This does all the usual startup stuff (start any SSH server, print a bunch of messages to the console, show a graphical login, etc). - -**Setting up the encrypted disk** - -First off, I used TWO hard drivesâthis is not an in-place migration, and that way nothing is broken if you mess up. One disk was in my root, and stayed there the whole time. The other I connected via USB. - -Hereâs the output of `gdisk -l` on my original disk: - -``` -Number Start (sector) End (sector) Size Code Name - 1 2048 1050623 512.0 MiB EF00 # EFI, mounted at /boot/efi - 2 1050624 354803711 168.7 GiB 8300 # ext4, mounted at / - 3 354803712 488396799 63.7 GiB 8200 # swap -``` - -Here will be the final output of `gdisk -l` on the new disk: - -``` -Number Start (sector) End (sector) Size Code Name - 1 2048 526335 256.0 MiB EF00 efi # EFI, mounted at /boot/efi - 2 1050624 135268351 64.0 GiB 8200 swap # swap - 3 135268352 937703054 382.6 GiB 8300 root_cipher # ext4-on-LUKS. ext4 mounted at / - 4 526336 1050623 256.0 MiB 8300 boot # ext4, mounted at /boot -``` - -1. Stop anything else running. Weâre going to do a âliveâ copy from the running system, so at least stop doing anything else. Also most of the commands in this guide need root (`sudo`). -2. Format the new disk. I used `gdisk` and you must select a gpt partition table. Basically I just made everything match the original. The one change I need is to add a /boot partition, so grub2 will be able to do the second stage. I also added partition labels with the `c` gdisk command to all partitions: boot, root\_cipher, efi, and swap. I decided Iâd like to be able to migrate to a larger disk later without updating a bunch of GUIDs, and filesystem or partition labels are a good method. -3. Add encryption. I like filesystem-on-LUKS, but most other debian guides use filesystem-in-LVM-on-LUKS. Youâll enter your new disk password twiceâonce to make an encrypted partition, once to open the partition. - `cryptsetup luksFormat /dev/disk/by-partlabel/root_cipher` - `cryptsetup open /dev/disk-by-partlabel/root_cipher root` -4. Make the filesystems. For my setup: - `mkfs.ext4 /dev/disk/by-partlabel/root` - `mkfs.ext4 /dev/disk/by-partlabel/boot` - `mkfs.vfat /dev/disk/by-partlabel/efi` -5. Mount all the new filesystems at `/mnt`. Make sure everything (cryptsetup included) uses EXACTLY the same mount paths (ex /dev/disk/by-partlabel/boot instead of /dev/sda1) as your final system will, because debian will examine your mounts to generate boot config files. - `mount /dev/disk/by-partlabel/root /mnt` - `mkdir /mnt/boot && mount /dev/disk/by-partlabel/boot /mnt/boo`t - `mkdir /mnt/boot/efi && mount /dev/disk/by-partlabel/efi /mnt/boot/efi` - `mkdir /mnt/dev && mount --bind /dev /mnt/dev # for chroot` - `mkdir /mnt/sys && mount --bind /sys /mnt/sys` - `mkdir /mnt/proc && mount --bind /dev /mnt/proc` -6. Copy everything over. I used `rsync -axAX`, but you can also use `cp -ax`. To learn what all these options are, read the man page. Make sure to keep the trailing slashes in the folder paths for rsync. - `rsync -xavHAX / /mnt/ --no-i-r --info=progress2` - `rsync -xavHAX /boot/ /mnt/boot/` - `rsync -xavHAX /boot/efi/ /mnt/boot/efi/` -7. Chroot in. You will now be âinâ the new system using your existing kernel. - `chroot /mnt` -8. Edit /etc/crypttab. Add: - `root PARTLABEL=root_cipher none luks` -9. Edit /etc/fstab. Mine looks like this: - `/dev/mapper/root / ext4 errors=remount-ro 0 1 PARTLABEL=boot /boot ext4 defaults,nofail 0 1 PARTLABEL=efi /boot/efi vfat umask=0077,nofail PARTLABEL=swap none swap sw,nofail 0 0 tmpfs /tmp tmpfs mode=1777,nosuid,nodev 0 0` -10. Edit /etc/default/grub. On debian you donât need to edit `GRUB_CMDLINE_LINUX`. - `GRUB_DISABLE_LINUX_UUID=true GRUB_ENABLE_LINUX_PARTLABEL=true` -11. Run `grub-install`. This will install the bootloader to efi. I forget the options to run it withâ¦ sorry! -12. Run `update-grub` (with no options). This will update /boot/grub.cfg so it knows how to find your new drive. You can verify the file by hand if you know how. -13. Run `update-initramfs` (with no options). This will update the initramfs so it can decrypt your root drive. -14. If there were any warnings or errors printed in the last three steps, something is wrong. Figure out whatâit wonât boot otherwise. Especially make sure your /etc/fstab and /etc/crypttab *exactly* match what youâve already used to mount filesystems. -15. Exit the chroot. Make sure any changes are synced to disk (you can unmount everything under /mnt in reverse order to make sure if you want) -16. Shut down your computer. Remove your root disk and boot from the new one. It should work now, asking for your password during boot. -17. Once you boot successfully and verify everything mounted, you can remove the `nofail` from /etc/fstab if you want. -18. (In my case, I also set up the swap partition after successful boot.) Edit: Oh, also donât use unencrypted swap with encrypted root. That was dumb. diff --git a/posts-md/money-orders.md b/posts-md/money-orders.md deleted file mode 100644 index 7007cb0..0000000 --- a/posts-md/money-orders.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -author: admin -categories: -- Non-Technical -date: 2014-12-01 12:19:47-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- finance -- hacks -title: Money orders -updated: 2014-12-02 13:51:00-07:00 -wordpress_id: 84 -wordpress_slug: money-orders ---- -[![alt:A postal money order](https://blog.za3k.com/wp-content/uploads/2014/12/Figure6-300x237.jpg)](https://blog.za3k.com/wp-content/uploads/2014/12/Figure6.jpg) - -A postal money order - -Allow me to introduce you all to the postal money order. For $1.50, you can get the equivalent of a cashierâs check from the post office. It can only be cashed by whoever you make it out to, and itâs basically accepted as cash by every corporation. You can also just give someone a blank one, although thatâs riskier to carry around for the obvious reasons. - -I was tired of checks bouncing. I canât be bothered to make sure my account remains such-and-such, which means it happens sometimes, especially times like now when Iâm poor. So I asked my landlord if I could pay by money orderâheâd never heard of them before, but seemed okay with it when I explained (heâs a really good guy!). - -I went down to the bank and got out $2750, and headed to the post office. I asked for 9 money orders, each for $303. The postal worker really only made a couple funny faces about me being weird, although my friend said she was prettyÂ loud about my walking out with that much cash-equivalent, it went pretty well. And I immediately endorsed all the money orders so now they can lie around the hose safely. - -Also, they come with attachable receipts (shown in the picture) in case you lose the check andÂ need a replacement, so thatâs nice. diff --git a/posts-md/postmortem-bs-store.md b/posts-md/postmortem-bs-store.md deleted file mode 100644 index ff417d2..0000000 --- a/posts-md/postmortem-bs-store.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -author: admin -categories: -- Technical -date: 2020-05-12 14:00:28-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- immutable -- linux -- postmortem -- storage -title: 'Postmortem: bs-store' -updated: 2020-05-17 12:52:32-07:00 -wordpress_id: 523 -wordpress_slug: postmortem-bs-store ---- -Between 2020-03-14 and 2020-12-03 I ran an experimental computer storage setup. I movied or copied 90% of my files into a content-addressable storage system. Iâm doing a writeup of why I did it, how I did it, and why I stopped. My hope is that it will be useful to anyone considering using a similar system. - -The assumption behind this setup, is that 99% of my files never change, so itâs fine to store only one, static copy of them. (Think movies, photosâ¦ theyâre most of your computer space, and youâre never going to modify them). There are files you change, I just didnât put them into this system. If you run a database, this ainât for you. - -Because I have quite a lot of files and 42 drives (7 in my computer, ~35 in a huge media server chassis), there is a problem of how to organize files across drives. To explain why itâs a problem, letâs look at the two default approaches: - -- **One Block Device / RAID 0**. Use some form of system that unifies block devices, such as RAID0 or a ZFSâs striped vdevs. Writing files is very easy, you see a single 3000GB drive. - - Many forms of RAID0 use striping. Striping splits each file across all available drives. 42 drives could spin up to read one file (wasteful). - - You need all the drives mounted to read anythingâI have ~40 drives, and Iâd like a solution that works if I move and canât keep my giant media server running. Also, itâs just more reassuring that nothing can fail if you can read each drive individually. -- **JBOD / Just a Bunch of Disks**. Label each drive with a category (ex. âmoviesâ), and mount them individually. - - Itâs hard to aim for 100% (or even >80%) drive use. Say you have 4x 1000GB drives, and you have 800GB movies, 800GB home video footage, 100GB photographs, and 300GB datasets. How do you arrange that? One drive per dataset is pretty wasteful, as everything fits on 3. But, with three drives, youâll need to split at least one dataset across drives. Say you put together 800GB home video and 100GB photographs. If you get 200GB more photographs, do you split a 300 GB collection across drives, or move the entire thing to another drive? Itâs a lot of manual management and shifting things around for little reason. - -Neither approach adds any redundancy, and 42 drives is a bit too many to deal with for most things. Step 1 is to split the 42 drives into 7 ZFS vdevs, each with 2-drive redundancy. That way, if a drive fails or there is a small data corruption (likely), everything will keep working. So now we only have to think about accessing 7 drives (but keep in mind, many physical drives will spin up for each disk access). - -The ideal solution: - -- Will not involve a lot of manual management -- Will fill up each drive in turn to 100%, rather than all drives at an equal %. -- Will deduplicate identical content (this is a ânice to haveâ) -- Will only involve accessing one drive to access one file -- Will allow me to get and remove drives, ideally across heterogenous systems. - -I decided a content-addressable system was ideal for me. That is, youâd look up each file by its hash. I donât like having an extra step to access files, so files would be accessed by symlinkâno frontend program. Also, it was important to me that I be able to transparently swap out the set of drives backing this. I wanted to make the content-addressable system basically a set of 7 content-addressable systems, and somehow wrap those all into one big content-addressable system with the same interface. Hereâs what I settled on: - -- (My drives are mounted as /zpool/bs0, /zpool/bs1, â¦ /zpool/bs6) -- Files will be stored in each pool in turn by hash. So my movie âcat.mpgâ with sha hash â8323f58d8b92e6fdf190c56594ed767df90f1b6dâ gets stored in /zpool/bs0/83/23/f58d8b9 \[shortened for readability\] -- Initially, we just copy files into the content-addressable system, we donât delete the original. Iâm cautious, and I wanted to make everything worked before getting rid of the originals. -- To access a file, I used read-only unionfs-fuse for this. This checks each of /zpool/bs{0..6}/ in turn. So in the final version, /data/movies/cat.mpg would be a symlink to â/bs-union/83/23/f58d8b9â -- We store some extra metadata on the original file (if not replaced by a symlink) and the storied copyâwhat collection itâs part of, when it was added, how big it is, what itâs hash is, etc. I chose to use xattrs. - -The plan here is that it would be really easy to swap out one backing blockstore of 30GB, for two of 20GBâjust copy the files to the new drives and add it to the unionfs. - -Hereâs what went well: - -- No problems during developmentâonly copying files meant it was easy and safe to debug prototypes. -- Everything was trivial to access (except see note about mounting disks below) -- It was easy to add things to the system -- Holding off on deleting the original content until I was 100% out of room on my room disks, meant it was easy to migrate off of, rather risk-free -- Running the entire thing on top of zfs ZRAID2 was the right decision, I had no worries about failing drives or data corruption, despite a lot of hardware issues developing at one point. -- My assumption that files would never change was correct. I made the unionfs filesystem read-only as a guard against error, but it was never a problem. -- Migrating off the system went smoothly - -Here are the implementation problems I found - -- I wrote the entire thing as bash scripts operating directly on files, which was OK for access and putting stuff in the store, but just awful for trying to get an overview of data or migrating things. I definitely should have used a database. I maybe should have used a programming language. -- Because there was no database, there wasnât really any kind of regular check for orphans (content in the blobstore with no symlinks to it), and other similar checks. -- unionfs-fuse suuucks. Every union filesystem Iâve tried sucks. Its read bandwidth is much lower than the component devices (unclear, probably), it doesnât cache where to look things up, and it has zero xattrs support (canât read xattrs from the underlying filesystem). -- gotcha: zfs xattrs waste a lot of space by default, you need to reconfigure the default. - -But the biggest problem was disk access patterns: - -- I thought I could cool 42 drives spinning, or at least a good portion of them. This was WRONG by far, and I am not sure how possible it is in a home setup. To give you an idea how bad this was, I had to write a monitor to shut off my computer if the drives went above 60C, and I was developing fevers in my bedroom (where the server is) from overheating. Not healthy. -- unionfs has to check each backing drive. So we see 42 drives spin up. I have ideas on fixing this, but it doesnât deal with the other problems -- To fix this, you could use double-indirection. - - Rather than pointing a symlink at a unionfs: /data/cat.mpg -> /bs-union/83/23/f58d8b9 (which accesses /zpool/bs0/83/23/f58d8b9) - - Point a symlink at another symlink that points directly to the data: /data/cat.mpg -> /bs-indirect/83/23/f58d8b9 -> /zpool/bs0/83/23/f58d8b9 -- The idea is that backing stores are kinda âwhatever, just shove it somewhereâ. But, actually it would be good to have a collection in one placeânot only to make it easy to copy, but to spin up only one drive when you go through everything in a collection. It might even be a good idea to have a separate drive for more frequently-accessed content. This wasnât a huge deal for me since migrating existing content meant it coincidentally ended up pretty localized. -- Because I couldnât spin up all 42 drives, I had to keep a lot of the array unmounted, and mount the drives I needed into the unionfs manually. - -So although I could have tried to fix things with double-indirection, I decided there were some other disadvantages to symlinks: estimating sizes, making offsite backups foolproof. I decided to migrate off the system entirely. The migration went well, although it required running all the drives at once, so some hardware errors popped up. Iâm currently on a semi-JBOD system (still on top of the same 7 ZRAID2 devices). - -Hopefully this is useful to someone planning a similar system someday. If you learned something useful, or there are existing systems I should have used, feel free to leave a comment. diff --git a/posts-md/scroll-props.md b/posts-md/scroll-props.md deleted file mode 100644 index a346f60..0000000 --- a/posts-md/scroll-props.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -author: admin -categories: -- Non-Technical -date: 2023-06-07 09:38:53-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- crafts -- dungeons and dragons -- feelies -title: Scroll Props -updated: 2023-06-07 09:38:53-07:00 -wordpress_id: 1040 -wordpress_slug: scroll-props ---- -Infocom introduced (AFAIK) the concept of feelies: - -> \[â¦\] Imaginative props and extras tied to the gameâs themeâprovidedÂ [copy protection](https://en.wikipedia.org/wiki/Copy_protection)Â againstÂ [copyright infringement](https://en.wikipedia.org/wiki/Copyright_infringement).[\[45\]](https://en.wikipedia.org/wiki/Infocom#cite_note-dyer19840506-45)Â Some games were unsolvable without the extra content provided with the boxed game. And because of the cleverness and uniqueness of the feelies, users rarely felt like they were an intrusion or inconvenience, as was the case with most of the other copy-protection schemes of the time.[\[49\]](https://en.wikipedia.org/wiki/Infocom#cite_note-49)Â Feelies also provided the player with a physical aspect to the gameplay of their text adventures, giving another dimension of strategy to what would other-wise just be a text parser. -> -> â Wikipedia (Infocom) - -I love to give out feelies for my D&D campaigns. Here are some lil handout props I made: - -[![](https://blog.za3k.com/wp-content/uploads/2023/06/scroll-crop-835x1024.jpg)](https://blog.za3k.com/wp-content/uploads/2023/06/scroll-crop.jpg) - -I used a receipt printer, q-tips, tape, and [orthodontic rubber bands](https://www.amazon.com/Orthodontic-Elastic-Rubberbands-Dreadlocks-Horse/dp/B00OSR1RBM). diff --git a/posts-md/the-double-lives-of-books.md b/posts-md/the-double-lives-of-books.md deleted file mode 100644 index f328352..0000000 --- a/posts-md/the-double-lives-of-books.md +++ /dev/null @@ -1,50 +0,0 @@ ---- -author: admin -categories: -- Non-Technical -- Technical -date: 2014-11-21 19:20:29-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: [] -title: The Double Lives of Books -updated: 2015-12-24 19:46:17-07:00 -wordpress_id: 46 -wordpress_slug: the-double-lives-of-books ---- -Two forces pull at me: the desire to have few possessions and be able to travel flexibly, and the convenience of reading and referencing physical books. I discovered a third option: I have digital copies of all my books, so I can freely get rid of them at any time, or travel without inconvenience. - -So thatâs where we start. Hereâs where I went. - -I thought, if these books are just a local convenience for an online version, itâs more artistically satisfying to have some representation of that. So I printed up a card catalog of all my books, both the ones I have digital copies of and not: - -[![alt:An example catalog card](https://blog.za3k.com/wp-content/uploads/2014/11/sample_card-300x186.png)](https://blog.za3k.com/wp-content/uploads/2014/11/sample_card.png) - -An example catalog card - -Thatâs what a card looks like. Thereâs information about the book up top, and a link in the form of a [QR code](http://en.wikipedia.org/wiki/QR_code "QR code") in the middle. The link downloads a PDF version of that book. Obviously being aÂ programmer, the cards all all automatically generated. - -[![alt:Book with a card inside](https://blog.za3k.com/wp-content/uploads/2014/11/book-186x300.jpg)](https://blog.za3k.com/wp-content/uploads/2014/11/book.jpg) - -Book with a card inside - -For the books where I have a physical copy, I put the card in the book, and it feels like Iâm touching the digital copy. My friends can pirate their own personal version of the book (saving me the sadness of lost lent-out books Iâm sure weâve all felt at times). And I just thing it looks darn neat.Â Some physical books I donât have a digital version of, since the world is not yet perfect. But at least I can identify them at a glance (and consider sending them off to a service likeÂ [http://1dollarscan.com/](http://1dollarscan.com/)) - -[![alt:Card catalog of digital books](https://blog.za3k.com/wp-content/uploads/2014/11/catalog-225x300.jpg)](https://blog.za3k.com/wp-content/uploads/2014/11/catalog.jpg) - -Card catalog of digital books - -And then, I have a box full of all the books I \*donât\* have a physical copy of, so I can browse through them, and organize them into reading lists or recommendations. Itâs not nearly as cool as the ones in books, but itâs sort of nice to keep around. - -And if I ever decide to get rid of a book, I can just check to make sure thereâs a card inside, and move the card into the box, reassured nothing is lost, giving away a physical artifact I no longer have the ability to support. - -I sadly wonât provide a link to the library since that stuff is mostly pirated. - -Interesting technical problems encountered during this project (you can stop reading now if youâre not technically inclined): - -- Making sure each card gets printed exactly once, in the face of printer failures and updating digital collections. This was hard and took up most of my time, but itâs also insanely boring so Iâll say no more. -- Command-line QR code generation, especially without generating intermediate files. I used rqrcode\_png in ruby. I can now hotlink link qr.png?text=Hello%20World and see any textÂ I want, itâs great. -- Printing the cards. This is actually really difficult to automateâI generate the cards in HTML and itâs pretty difficult to print HTML, CSS, and included images. I ended up using the â[wkhtmltoimage](http://wkhtmltopdf.org/ "wkhtmltoimage")â project, which as far as I can tell, renders the image somewhere internally using [webkit](http://en.wikipedia.org/wiki/WebKit "webkit") and screenshots it. Thereâs also a wkhtmltopdf available, which worked well but I couldnât get to cooperate with index-card sized paper. Nothing else really seems to handle CSS, etc properly and as horrifying as the fundamental approach is, itâs both correct and well-executed. (They solved a number of problems with upstream patches to Qt for example, the sort of thing I love to hear) -- The [zbarcam](http://zbar.sourceforge.net/ "zbarcam") software (for scanning QR codes among other digital codes) is just absolute quality work and I canât say enough good things about it. Scanning cards back into the computer was one of the most pleasant parts of this whole project. It has an intuitive command UI using all the format options I want, and camera feedback to show itâs scanned QR codes (which it does very quickly). -- [Future-proofed](http://en.wikipedia.org/wiki/Future_proof "Future-proofed") links to pirated booksâthe sort of link that usually goes down. I opted to use a [SHA256 hash](http://en.wikipedia.org/wiki/SHA-2 "SHA256 hash") (the mysterious numbers at the bottom which form a unique signature generated from the content of the book) and provide a small page on my website which gives you a download based on that. This is what the QR code links to. I was hoping there was some way to provide that without involving me, but Iâm unaware of anyÂ service available. [Alice Monday](https://twitter.com/ali0mt "Alice Monday") suggested just typing the SHA hash into Google, which sounded like the sort of clever idea which might work. It doesnât. diff --git a/posts-md/understanding-gzip-2.md b/posts-md/understanding-gzip-2.md deleted file mode 100644 index 886c6db..0000000 --- a/posts-md/understanding-gzip-2.md +++ /dev/null @@ -1,370 +0,0 @@ ---- -author: admin -categories: -- Technical -date: 2021-07-10 21:36:51-07:00 -has-comments: true -markup: markdown -source: wordpress -tags: -- compression -- formats -- gzip -- informative -title: Understanding gzip -updated: 2021-07-11 18:25:09-07:00 -wordpress_id: 668 -wordpress_slug: understanding-gzip-2 ---- -Letâs take a look at the gzip format. Why might you want to do this? - -1. Maybe youâre curious how gzip works -2. Maybe youâre curious how DEFLATE works. DEFLATE is the âactualâ compression method inside of gzip. Itâs also used in zip, png, git, http, pdfâ¦ the list is pretty long. -3. Maybe you would like to write a gzip/DEFLATE decompressor. (A compressor is more complicatedâunderstanding the format alone isnât enough) - -Letâs work a few examples and look at the format in close detail. For all these examples, Iâm using GNU gzip 1.10-3 on an x86\_64 machine. - -I recommend checking out the linked resources below for a deeper conceptual overview if you want to learn more. That said, these are the only worked examples of gzip and/or DEFLATE of which Iâm aware, so theyâre a great companion to one another. In particular, you may want to learn what a prefix code is ahead of time. - -References: -\[1\] [RFC 1951](https://www.rfc-editor.org/rfc/rfc1951.txt), DEFLATE standard, by Peter Deutsch -\[2\] [RFC 1952](https://www.rfc-editor.org/rfc/rfc1952.txt), gzip standard, by Peter Deutsch -\[3\] [infgen](https://github.com/madler/infgen), by Mark Adler (one of the zlib/gzip/DEFLATE authors), a tool for dis-assembling and printing a gzip or DEFLATE stream. I found this useful in figuring out the endian-ness of bitfields, and somewhat in understanding the dynamic huffman decoding process. Documentation is [here](https://github.com/madler/infgen/blob/master/infgen.c). -\[4\] [An explanation of the âdeflateâ algorithm](https://zlib.net/feldspar.html) by Antaeus Feldspar. A great conceptual overview of LZ77 and Huffman coding. **I recommend reading this *before* reading my DEFLATE explanation.** -\[5\] [LZ77](https://en.wikipedia.org/wiki/LZ77_and_LZ78) compression, Wikipedia. -\[6\] [Prefix-free codes](https://en.wikipedia.org/wiki/Prefix_code) generally and [Huffman](https://en.wikipedia.org/wiki/Huffman_coding)âs algorithm specifically -\[7\] After writing this, I learned about [puff.c](https://github.com/madler/zlib/blob/master/contrib/puff/puff.c), a reference (simple) implementation of a DEFLATE decompressor by Mark Adler. - -## Gzip format: Basics and compressing a stream - -Letâs take a look at our first example. If youâre on Linux, feel free to run the examples I use as we go. - -``` -echo "hello hello hello hello" | gzip -``` - -The bytes gzip outputs are below. You can use *xxd* or any other hex dump tool to view binary files. Notice that the original is 24 bytes, while the compressed version is 29 bytesâgzip is not really intended for data this short, so all of the examples in this article actually get bigger. - -| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| Byte | **0** | **1** | **2** | **3** | **4** | **5** | **6** | **7** | **8** | **9** | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | **21** | **22** | **23** | **24** | **25** | **26** | **27** | **28** | -| Hex | **1f** | **8b** | **08** | **00** | **00** | **00** | **00** | **00** | **00** | **03** | cb | 48 | cd | c9 | c9 | 57 | c8 | 40 | 27 | b9 | 00 | **00** | **88** | **59** | **0b** | **18** | **00** | **00** | **00** | - -hello (1) â gzip contents - -The beginning and end in bold are the gzip header and footer. I learned the details of the format by reading [RFC 1952: gzip](https://www.rfc-editor.org/rfc/rfc1952.txt) - -- Byte 0+1 (1f8b): Two fixed bytes that indicate âthis is a gzip fileâ. These file-type indicators are also called âmagic bytesâ. -- Byte 2 (08): Indicates âthe compression format is DEFLATEâ. DEFLATE is the only format supported by gzip. -- Byte 3 (00): Flags. 8 single-bit flags. - - Not set: TEXT (indicates this is ASCII text. hint to the decompressor only. i think gzip never sets this flag) - - Not set: HCRC (adds a 16-bit CRC to the header) - - Not set: EXTRA (adds an âextrasâ field to the header) - - Not set: NAME (adds a filename to the headerâif you compress a file instead of stdin this will be set) - - Not set: COMMENT (adds a comment to the header) - - There are also three reserved bits which are not used. -- Byte 4-7 (00000000): Mtime. These indicate when the compressed file was last modified, as a unix timestamp. gzip doesnât set an associated time when compressing stdin. Technically the standard says it should use the current time, but this makes the output the same every time you run gzip, so itâs better than the original standard. -- Byte 8 (00): Extra flags. 8 more single-bit flags, this time specific to the DEFLATE format. None are set so letâs skip it. All they can indicate is âminimum compression levelâ and âmax compression levelâ. -- Byte 9 (03): OS. OS â03â is Unix. -- Byte 10-20: Compressed (DEFLATE) contents. Weâll take a detailed look at DEFLATE below. -- Byte 21-24 (0088590b): CRC32 of the uncompressed data, âhello hello hello hello\\nâ. I assume this is correct. Itâs worth noting, there are multiple things called âCRC32â. -- Byte 25-28 (18000000): Size of the uncompressed data. This is little-endian byte order, 0x00000018 = 16\*1+1\*8 = 24. The uncompressed text is 24 bytes, so this is correct. - -| | | | | | | | | | | | | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| Byte | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -| Hex | cb | 48 | cd | c9 | c9 | 57 | c8 | 40 | 27 | b9 | 00 | -| Binary | 11001011 | 01001000 | 11001101 | 11001001 | 11001001 | 01010111 | 11001000 | 01000000 | 00100111 | 10111001 | 00000000 | -| R. Bin. | 11010011 | 00010010 | 10110011 | 10010011 | 10010011 | 11101010 | 00010011 | 00000010 | 11100100 | 10011101 | 00000000 | - -hello (1) â DEFLATE contents - -## DEFLATE format: Basics and fixed huffman coding - -DEFLATE is the actual compression format used inside gzip. The format is detailed in [RFC 1951: DEFLATE](https://www.rfc-editor.org/rfc/rfc1951.txt). DEFLATE is a dense format which uses bits instead of bytes, so we need to take a look at the binary, not the hex, and things will not be byte-aligned. The endian-ness is a little confusing in gzip, so weâll usually be looking at the âreversed binaryâ row. - -- As a hint, whenever we read bits, we use the âreverseâ binary order. For Huffman codes, we keep the bit order in reverse. For fixed-length fields like integers, we reverse again into ânormalâ binary order. Iâll call out the order for each field. -- Byte 10: **1** 1010011. Is it the last block? Yes. - - 1: Last block. The last block flag here means that after this block ends, the DEFLATE stream is over -- Byte 10: 1 **10** 10011\. Fixed huffman coding. We reverse the bits (because itâs always 2 bits, and we reverse any fixed number of bits) to get 01. - - - 00: Not compressed - - - **01: Fixed huffman coding.** - - 10: Dynamic huffman coding. - - 11: Not allowed (error) -- So weâre using âfixedâ huffman coding. That means thereâs a static, fixed encoding scheme being used, defined by the DEFLATE standard. The scheme is given by the tables below. Note that Length/Distance codes are specialâafter you read one, you may read some extra bits according to the length/distance lookup tables. - -| Binary | Bits | Extra bits | Type | Code | -| --- | --- | --- | --- | --- | -| 00110000-10111111 | 8 | 0 | Literal byte | 0-143 | -| 110010000-111111111 | 9 | 0 | Literal byte | 144-255 | -| 0000000 | 7 | 0 | End of block | 256 | -| 0000001-0010111 | 7 | varies | Length | 257-279 | -| 11000000-11000111 | 8 | varies | Length | 280-285 | - -Literal/End of Block/Length Huffman codes - -| Binary Code | Bits | Extra bits | Type | Value | -| --- | --- | --- | --- | --- | -| 00000-111111 | 5 | varies | Distance | 0-31 | - -Distance Huffman codes - -| Code | Binary | Meaning | Extra bits | -| --- | --- | --- | --- | -| 267 | 0001011 | Length 15-16 | 1 | - -Length lookup (abridged) - -| Code | Binary | Meaning | Extra bits | -| --- | --- | --- | --- | -| 4 | 00100 | Distance 5-6 | 1 | - -Distance lookup (abridged) - -- Now we read a series of codes. Each code might be - - a literal (one binary byte), which is directly copied to the output - - âend of blockâ. either another block is read, or if this was the last block, DEFLATE stops. - - a length-distance pair. first code is a length, then a distance is read. then some of the output is copiedâthis reduces the size of repetitive content. the compressor/decompressor can look up to 32KB backwards for duplicate content. This copying scheme is called [LZ77](https://en.wikipedia.org/wiki/LZ77_and_LZ78). -- Huffman codes are a âprefix-free codeâ (confusingly also called a âprefix codeâ). What that means is that, even though the code words are different lengths from one another, you can always unambigously tell which binary *codeword* is next. For example, suppose the bits youâre reading starts with: 0101. Is the next binary codeword 0, 01, 010, or 0101? In a prefix-free code, only one of those is a valid codeword, so itâs easy to tell. You donât need any special separator to tell you the codeword is over. The space savings from not having a separator is really important for good compression. The âhuffmanâ codes used by DEFLATE are prefix-free codes, but theyâre not really optimal Huffman codesâitâs a common misnomer. -- Byte 10-11: 110 **10011000** 10010: A literal. 10011000 (152) minus 00110000 (48) is 104. 104 in ASCII is âhâ. -- Byte 11-12: 000 **10010101** 10011: A literal. 10010101 (149) minus 00110000 (48) is 101. 101 in ASCII is âeâ. -- Byte 12-13: 101 **10011100** 10011: A literal. 10011100 (156) minus 00110000 (48) is 108. 108 in ASCII is âlâ. -- Byte 13-14: 100 **10011100** 10011: Another literal âlâ -- Byte 14-15: 100 **10011111** 01010: A literal. 10011111 (159) minus 00110000 (48) is 111. 111 in ASCII is âoâ. -- Byte 15-16: 111 **01010000** 10011: A literal. 01010000 (80) minus 00110000 (48) is 32. 32 in ASCII is â â (space). -- Byte 16-17: 000 **10011000** 00010: Another literal âhâ. -- Byte 17: 000 **0001011**: A length. 0001011 (11) minus 0000001 (1) is 10, plus 257 is 267. We look up distance 256 in the âlength lookupâ table. The length is 15-16, a range. -- Byte 18: **1** 00100: Because the length is a range, we read extra bits. The âlength lookupâ table says to read 1 extra bit: 1. The extra bits need to be re-flipped back to normal binary order to decode them, but 0b1 flipped is still 0b1. 15 (bottom of range) plus 0b1 = 1 (extra bits) is 16, so the final length is 16. -- Byte 18-19: 111 **00100** 10011101: After a length, we always read a distance next. Distances are encoded using a second huffman table. 00100 is code 4, which using the âdistance lookupâ table is distance 5-6. -- Byte 18-19: 11100100 **1** 0011101. Using the âdistance lookupâ table, we need to read 1 extra bit: 0b1. Again, we reverse it, and add 5 (bottom end of range) to 0b1 (extra bits read), to get a distance of 6. -- We copy from 6 characters ago in the output stream. The stream so far is âhello hâ, so 6 characters back is starting at âeâ. We copy 16 characters, resulting in âhello h**ello hello hello**â. Why this copy didnât start with the second âhâ instead of the second âeâ, Iâm not sure. -- Byte 19-20: 1 **00111010** 0000000: A literal. 00111010 (58) minus 00110000 (48) is 10. 10 in ASCII is â\\nâ (new line) -- Byte 20: 0 **0000000**: End of block. In this case we ended nicely on the block boundry, too. This is the final block, so weâre done decoding entirely. -- At this point weâd check the CRC32 and length match whatâs in the gzip footer right after the block. - -Our final output is âhello hello hello hello\\nâ, which is exactly what we expected. - -## Gzip format: Compressing a file - -Letâs generate a second example using a file. - -``` -echo -en "\xff\xfe\xfd\xfc\xfb\xfa\xf9\xf8\xf7\xf6\xf5\xf4\xf3\xf2\xf1" >test.bin -gzip test.bin -``` - -This input file is pretty weird. In fact, itâs so weird that gzip compression will fail to reduce its size at all. Weâll take a look at what happens when compression fails in the next DEFLATE section below. But first, letâs see how gzip changes with a file instead of a stdin stream. - -| | | | | | | | | | | | | | | | | | | | | | | | | | | | | | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| Byte | 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19-38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -| Hex | 1f | 8b | 08 | 08 | 9f | 08 | ea | 60 | 00 | 03 | 74 | 65 | 73 | 74 | 2e | 62 | 69 | 6e | 00 | see below | c6 | d3 | 15 | 7e | 0f | 00 | 00 | 00 | - -binary garbage (2) â abridged gzip contents - -Okay, letâs take a look at how the header and footer changed. - -- Byte 0+1 (1f8b): Two fixed bytes that indicate âthis is a gzip fileâ. These file-type indicators are also called âmagic bytesâ. -- Byte 2 (08): Indicates âthe compression format is DEFLATEâ. DEFLATE is the only format supported by gzip. -- Byte 3 (08): Flags. 8 single-bit flags. - - Not set: TEXT (indicates this is ASCII text. hint to the decompressor only. i think gzip never sets this flag) - - Not set: HCRC (adds a 16-bit CRC to the header) - - Not set: EXTRA (adds an âextrasâ field to the header) - - **Set: NAME** (adds a filename to the header) - - Not set: COMMENT (adds a comment to the header) - - There are also three reserved bits which are not used. -- **Byte 4-7 (9f08ea60) . Mtime.** This is in little-endian order: 0x60ea089f is 1625950367. This is a unix timestamp â 1625950367 seconds after midnight, Jan 1, 1970 is 2021-07-10 20:52:47 UTC, which is indeed earlier today. This is the time the original file was last changed, not when compression happened. This is here so we can restore the original modification time if we want. -- Byte 8 (00): Extra flags. None are set. -- Byte 9 (03): OS. OS â03â is Unix. -- **Byte 10-18 (74 65 73 74 2e 62 69 6e 00): Zero-terminated string.** The string is âtest.binâ, the name of the file to decompress. We know this field is present because of the flag set. -- Byte 19-38: The compressed DEFLATE stream. -- **Byte 39-42 (c6d3157e): CRC32 of the uncompressed data.** Again, Iâll just assume this is correct. -- **Byte 25-28 (0f000000): Size of the uncompressed data.** 0x0000000f = 15 bytes, which is correct. - -## DEFLATE format: Uncompressed data - -Uncompressed data is fairly rare in the wild from what Iâve seen, but for the sake of completeness weâll cover it. - -| | | | | | | | -| --- | --- | --- | --- | --- | --- | --- | -| Byte | 19 | 20 | 21 | 22 | 23 | 24-38 | -| Hex | 01 | 0f | 00 | f0 | ff | ff fe fd fc fa f9 f8 f7 f6 f5 f4 f3 f2 f1 | -| Binary | 00000001 | 00001111 | 00000000 | 11110000 | 11111111 | omitted | -| R. Binary | 10000000 | 11110000 | 00000000 | 00001111 | 11111111 | omitted | - -binary garbage (2) â DEFLATE contents - -- Again, we start reading âr. binaryâ â the binary bits in reversed order. -- Byte 19: **1**0000000\. The first three bits are the most important bits in the stream: - - 1: Last block. The last block flag here means that after this block ends, the DEFLATE stream is over -- Byte 19: 1**00** **00000**. Not compressed. For a non-compressed block only, we also skip until the end of the byte. - - - **00: Not compressed** - - - 01: Fixed huffman coding. - - 10: Dynamic huffman coding. -- Byte 20-21: **11110000 00000000**. Copy 15 uncompressed bytes. We reverse the binary bits as usual for fixed fields. 0b0000000000001111 = 0x000f = 15. -- Byte 22-23: **00001111 11111111.** This is just the NOT (compliment) of byte 20-21 as a check. It can be ignored. -- Byte 24-38: **ff fe fd fc fb fa f9 f8 f7 f6 f5 f4 f3 f2 f1**: 15 literal bytes of data, which are directly copied to the decompressed output with no processing. Since we only have one block, this is the whole of the decompressed data. - -## DEFLATE format: Dynamic huffman coding - -Dynamic huffman coding is by far the most complicated part of the DEFLATE and gzip specs. It also shows up a lot in practice, so we need to learn this too. Letâs take a look with a third and final example. - -``` -echo -n "abaabbbabaababbaababaaaabaaabbbbbaa" | gzip -``` - -The bytes we get are: - -- Byte 0-9 (**1f 8b 08 00 00 00 00 00 00 03**): Header -- Byte 10-32 (1d c6 49 01 00 00 10 40 c0 ac a3 7f 88 3d 3c 20 2a 97 9d 37 5e 1d 0c): DEFLATE contents -- Byte 33-40 (**6e 29 34 94 23 00 00 00**): Footer. The uncompressed data is 35 bytes. - -Weâve already seen everything interesting in the gzip format, so weâll skip the header and footer, and move straight to looking at DEFLATE this time. - -| | | | | | | | | | | | | | | | | | | | | | | | | -| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| Byte | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -| Hex | 1d | c6 | 49 | 01 | 00 | 00 | 10 | 40 | c0 | ac | a3 | 7f | 88 | 3d | 3c | 20 | 2a | 97 | 9d | 37 | 5e | 1d | 0c | -| Binary | 00011101 | 11000110 | 01001001 | 00000001 | 00000000 | 00000000 | 00010000 | 01000000 | 11000000 | 10101100 | 10100011 | 01111111 | 10001000 | 00111101 | 00111100 | 00100000 | 00101010 | 10010111 | 10011101 | 00110111 | 01011110 | 00011101 | 00001100 | -| R. Binary | 10111000 | 01100011 | 10010010 | 10000000 | 00000000 | 00000000 | 10000000 | 00000010 | 00000011 | 00110101 | 11000101 | 11111110 | 00010001 | 10111100 | 00111100 | 00000100 | 01010100 | 11101001 | 10111001 | 11101100 | 01111010 | 10111000 | 00110000 | - -abaa stream â DEFLATE contents - -- As usual, we read âr. binaryâ â the binary bits in reversed order. -- Byte 10: **1**0111000\. Last (only) block. The DEFLATE stream is over after this block. -- Byte 10: 1**01**11000\. 10=Dynamic huffman coding - - - 00: Not compressed - - - 01: Fixed huffman coding. - - **10: Dynamic huffman coding.** - -## Which parts are dynamic? - -Okay, so what does âdynamicâ huffman coding mean? A fixed huffman code had several hardcoded values defined by the spec. Some are still hardcoded, but some will now be defined by the gzip file. - -1. The available literals are all single-byte literals. **The literals remain fixed by the spec.** -2. There is a âspecialâ literal indicating the end of the block in both. -3. The lengths (how far to look backwards when copying) were given as ranges whose size was a power of two. For example, there would be one binary code (0001011) for the length range 15-16. Then, we would read one extra bit (because the range is 2^1 elements long) to find the specific length within that range. **In a dynamic coding, the length ranges remain fixed by the spec.** (âlength lookupâ table) -4. Again, the actual ranges and literals are fixed by the spec. **The binary codewords to represent (lengths/literals/end-of-block) are defined in the gzip stream instead of hardcoded.** (âliteral/end-of-block/length huffman codesâ table) -5. Like the literal ranges, **the distance ranges remain fixed by the spec.** (âdistance lookupâ table) -6. Although the distance ranges themselves are fixed, **the binary codewords to represent distance ranges are defined in the gzip stream instead of hardcoded.** (âdistance huffman codesâ table) - -So basically, the possible lengths and distances are still the same (fixed) ranges, and the literals are still the same fixed literals. But where we had two hardcoded tables before, now we will load these two tables from the file. Since storing a table is bulky, the DEFLATE authors heavily compressed the representation of the tables, which is why dynamic huffman coding is so complicated. - -## Aside: Storing Prefix-Free Codewords as a List of Lengths - -Suppose we have a set of prefix-free codewords: 0, 10, 1100, 1101, 1110, 1111. Forget about what each codeword means for a second, weâre just going to look at the codewords themselves. - -We can store the lengths as a list: 1, 2, 4, 4, 4, 4. - -- You could make another set of codewords with the same list of lengths. But for our purposes, as long as each value gets the same length of codeword, we donât really care which of those codes we pickâthe compressed content will be the same length. -- Since we donât really care how if the bits change, any code is fine. For simplicity, we pick a unique âstandardâ code. When we list the codewords, the standard one can be listed BOTH in order of length, AND in normal sorted order. That is, the those two orders are the same. The example code above is a standard code. Hereâs one that isnât: 1, 01, 00. -- It turns out that if we have the lengths, we can generate a set of prefix-free codewords with those lengths. Thereâs an easy algorithm to generate the âstandardâ code from the list of lengths (see RFC 1951, itâs not very interesting) -- Since we picked the standard codewords, we can switch back and forth between codewords and codeword lengths without losing any information. -- Itâs more compact to store the codeword lengths than the actual codewords. DEFLATE just stores codeword lengths everywhere (and uses the corresponding generated code). - -Finally, we need to make them correspond to symbols, so we actually store - -- We store lengths for each symbol: A=4, B=1, C=4, D=4, E=2, F=4 -- We can get the correct codewords by going through the symbols in order, and grabbing the first available standard codeword: A=1100, B=0, C=1101, D=1110, E=10, F=1111. - -## Dynamic Huffman: Code Lengths - -Whatâs a âcode lengthâ? Itâs yet another hardcoded lookup table, which explains how to compress the dynamic huffman code tree itself. Weâll get to it in a secondâthe important thing about it for now is that there are 19 rows in the table. The binary column (not yet filled in) is what weâre about to decode. - -| | | | | -| --- | --- | --- | --- | -| Binary | Code | What it means | Extra bits | -| ? | 0-15 | Code length 0-15 | 0 | -| ? | 16 | Copy the previous code length 3-6 times | 2 | -| ? | 17 | Copy â0â code length 3-10 times | 3 | -| ? | 18 | Copy â0â code length 11-138 times | 7 | - -Code Lengths (static) - -- Byte 10: 101 **11000**. Number of literal/end-of-block/length codes (257-286). Read bits in forward order, 0b00011=3, plus 257 is 260 length/end-of-block/literal codes. -- Byte 11: **01100** 011. Number of distance codes (1-32). Read bits in forward order, 0b00110=6, plus 1 is 7 distance codes. -- Byte 11-12: 01100 **0111** 0010010\. Number of *code length codes* used (4-19). Read bits in forward order, 0b1110=14, plus 4 is 18. -- Byte 12-18 1 **001 001 010 000 000 000 000 000 000 000 000 001 000 000 000 100 000 001** 1: There are 18 codes used (out of 19 available). For each code length code, we read a 3-bit number (in big-endian order) called the âcode length code lengthâ, and fill the end with 0s: 4, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 4\[, 0\] -- Next, we re-order the codes in the order 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15. This re-order is just part of the spec, donât ask whyâitâs to save a small amount of space. - The old order is: 16: 4, 17: 4, 18: 2, 0:0, 8:0, 7:0, 9:0, 6:0, 10:0, 5:0, 11:0, 4:4, 12:0, 3:0, 13:0, 2:1, 14:0, 1:4, 15:0 - The new order is: 0:0, 1:4, 2:1, 3:0, 4:4, 5:0, 6:0, 7:0, 8:0, 9:0, 10:0, 11:0, 12:0, 13:0, 14:0, 15:0, 16: 4, 17: 4, 18: 2 -- 0s indicate the row is not used (no codeword needed). Letâs re-write it without those. - 1:4, 2:1, 4:4, 16: 4, 17: 4, 18: 2 -- Now we assign a binary codewords of length N, to each length N in the list. - 1:1100,2:0,4:1101,16:1110,17:1111,18:10 -- Finally, letâs take a look at the whole table again. - -| Binary | Code | What it means | Extra bits | -| --- | --- | --- | --- | -| 1100 | 1 | Code length 1 | 0 | -| 0 | 2 | Code length 2 | 0 | -| 1101 | 4 | Code length 4 | 0 | -| 1110 | 16 | Copy the previous code length 3-6 times | 2 | -| 1111 | 17 | Copy â0â code length 3-10 times | 3 | -| 10 | 18 | Copy â0â code length 11-138 times | 7 | - -Code Lengths - -- Great, weâve parsed the code lengths table. - -## Dynamic Huffman: Parsing the Huffman tree - -- As a reminder, in bytes 10-12 we found there was a 260-row literal/end-of-block/length table and a 7-row distance table. Letâs read 267 numbers: the lengths of the codeword for each row. -- Byte 18-19: 0000001 **10 0110101**. Copy â0â code length 11-138 times - 0b1010110=86, plus 11 is 97. Literals 0-96 are not present. -- Byte 20: **1100** 0101: Literal 1. Literal 97 (âaâ) has a codeword of length 1. -- Byte 20: 1100 **0** 101: Literal 2. Number 98 (âbâ) has a codeword of length 2. -- Byte 20-21: 11000 **10 1111111** 10. Copy â0â code length 11-138 times. 0b1111111=127, plus 11 is 138. Literals 99-236 are not present. -- Byte 21-22: 111111 **10** **0001000** 1. Copy â0â code length 11-138 times. 0b0001000=8, plus 11 is 19. Literals 237-255 are not present. -- Bytes 22-23: 0001000 **1101** 11100\. Literal 256 (end-of-block) has a codeword of length 4. -- Byte 23-24: 101 **1110** **00** 0111100. Copy previous code 3-6 times. 0b00=0, plus 3 is 3. âLiteralsâ 257-259 (all lengths) have codewords of length 4. -- We read 260 numbers, thatâs the whole literal/end-of-block/length table. Assign the âstandardâ binary codewords based on the lengths to generate the following table: - -| Literal Code | Code Length | Binary | Meaning | Extra bits | -| --- | --- | --- | --- | --- | -| 97 | 1 | 0 | Literal âaâ | 0 | -| 98 | 2 | 10 | Literal âbâ | 0 | -| 256 | 4 | 1100 | End-of-block | 0 | -| 257 | 4 | 1101 | Length 3 | 0 | -| 258 | 4 | 1110 | Length 4 | 0 | -| 259 | 4 | 1111 | Length 5 | 0 | - -abaa dynamic literal/end-of-block/length Huffman codes - -- Now we read 7 more numbers in the same format: the 7-row distances table. -- Byte 24: 0 **0** 111100\. Distance 0 has a codeword of length 2. -- Byte 24-25: 00 **1111** **000** 0000100. Copy â0â code length 3-10 times. 0b000=0, plus 3 is 3. Distances 1-3 are not present. -- Byte 25: 0 **0 0 0** 0100: Distances 4-6 have length 2. -- We read 7 numbers, thatâs the whole distances table. Assign the âstandardâ binary codewords to generate the following table: - -| Code | Bits | Binary | Meaning | Extra Bits | -| --- | --- | --- | --- | --- | -| 0 | 2 | 00 | Distance 1 | 0 | -| 4 | 2 | 01 | Distance 5-6 | 1 | -| 5 | 2 | 10 | Distance 7-8 | 1 | -| 6 | 2 | 11 | Distance 9-12 | 2 | - -abaa dynamic literal/end-of-block/length Huffman codes - -## Dynamic Huffman: Data stream decoding - -- Now weâre ready to actually decode the data. Again, weâre reading a series of codes from the literal/end-of-block/length Huffman code table. -- Byte 25: 0000**0 10 0**: Literal âaâ, âbâ, âaâ -- Byte 26: **0** **10** **10** **10** **0**: Literal âaâ, âbâ, âbâ, âbâ, âaâ. -- Byte 27: **1110 10** **0** 1. Length 4. Whenever we read a length, we read a distance. The distance is a range, 7-8. The extra bit we read is 0b0=0, plus 7 is Distance 7. So we look back 7 bytes and copy 4. The new output is: baabbba**baab** -- Byte 27-28: 1110100 **1101** **11** **00** 1: Length 3, Distance 9. We look back 9 bytes and copy 3. The new output is: abbabaab**abb** -- Byte 28-29: 1011100 **1111** **01** **1** 00. Length 5, Distance 6. We look back 6 bytes and copy 5. The new output is: aababb**aabab** -- Byte 29: 111011 **0 0**. Literal âaâ, âaâ. -- Byte 30: **0** 1111010. Literal âaâ. -- Byte 30: 0 **1111 01** **0**. Length 5, Distance 5. We look back 5 bytes and copy 5. The new output is: abaaa**abaaa** -- Byte 31: **10** 111000: Literal âbâ -- Byte 31: 10 **1110** **00**: Length 4, Distance 1. We look back 1 byte and copy 4. The new output is: b**bbbb** -- Byte 32: **0 0** 110000: Literal âaâ, âaâ. -- Byte 32: 00 **1100** **00**: End-of block. Since this is the final block itâs also the end of the stream. This didnât come up in the first example, but we zero-pad until the end of the byte when the block ends. -- The final output is a b a a b b b a baab abb aabab a a a abaaa b bbbb a a (spaces added for clarity), which is exactly what we expected. diff --git a/posts-md/whiteboard-partition.md b/posts-md/whiteboard-partition.md deleted file mode 100644 index 21e52c8..0000000 --- a/posts-md/whiteboard-partition.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -author: admin -categories: -- Non-Technical -date: 2015-04-30 03:12:07-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- carpentry -- lost purposes -title: Whiteboard Partition -updated: 2015-04-30 03:12:07-07:00 -wordpress_id: 201 -wordpress_slug: whiteboard-partition ---- -I wanted a partition to divide my room, and I had a whiteboard sitting around. I sawed it into three parts, and connected them with hinges: - -![alt:whiteboard 2](https://blog.za3k.com/wp-content/uploads/2015/04/whiteboard-2.jpg) - -folding whiteboard - -![alt:whiteboard 4](https://blog.za3k.com/wp-content/uploads/2015/04/whiteboard-4.jpg) - -hinge (front) - -![alt:whiteboard 1](https://blog.za3k.com/wp-content/uploads/2015/04/whiteboard-1.jpg) - -hinge (back) - -Iâm a little embarrassed at having done all this, since it was obvious as soon as I started the partition was way too short to work. I figured Iâd still get some experience woodworking (this is my first project). Hereâs where it went: - -![alt:whiteboard 5](https://blog.za3k.com/wp-content/uploads/2015/04/whiteboard-5.jpg) - -and I never saw it again diff --git a/posts-md/whoosh.md b/posts-md/whoosh.md deleted file mode 100644 index 80f6933..0000000 --- a/posts-md/whoosh.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -author: admin -categories: -- Non-Technical -date: 2015-10-10 18:17:34-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- art -- piskell -- pixel art -title: Whoosh! -updated: 2015-10-17 19:15:58-07:00 -wordpress_id: 282 -wordpress_slug: whoosh ---- -[![alt:Action Potato](https://blog.za3k.com/wp-content/uploads/2015/10/action-potato-300x300.png)](https://blog.za3k.com/wp-content/uploads/2015/10/action-potato.png) - -Action Potato - -Itâs whooshing because itâs going as fast as a WEAK SPEEDBOAT. diff --git a/posts-md/wip-dead-tree-publishing-3.md b/posts-md/wip-dead-tree-publishing-3.md deleted file mode 100644 index 0201798..0000000 --- a/posts-md/wip-dead-tree-publishing-3.md +++ /dev/null @@ -1,27 +0,0 @@ ---- -author: admin -categories: -- Non-Technical -date: 2015-04-16 18:47:16-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- bootstrap -- dead tree -- publishing -- website -title: 'WIP: Dead Tree Publishing 3' -updated: 2015-04-16 18:58:51-07:00 -wordpress_id: 167 -wordpress_slug: wip-dead-tree-publishing-3 ---- -Compared with [last update](https://blog.za3k.com/wip-dead-tree-publishing-2/ "WIP: Dead Tree Publishing 2"), Â the Dead Tree Publishing website is looking nicer. - -[![alt:Looking better](https://blog.za3k.com/wp-content/uploads/2015/04/2015-04-16-184416_1366x768-e1429235272387.jpg)](https://blog.za3k.com/wp-content/uploads/2015/04/2015-04-16-184416_1366x768.jpg) - -Looking better - -Itâs served over HTTPS now (not needed for security, but it puts people at ease and enabled Chromeâs autocomplete) and you can order multiple books at a time. - -Other than some more visual improvements, the main thing missing is proper detection of page size â my server doesnât understand about page margins, so it things books are bigger than they really are. diff --git a/posts-md/wip-dead-tree-publishing-4.md b/posts-md/wip-dead-tree-publishing-4.md deleted file mode 100644 index d90cdb6..0000000 --- a/posts-md/wip-dead-tree-publishing-4.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -author: admin -categories: -- Non-Technical -date: 2015-04-30 18:08:01-07:00 -has-comments: true -markup: markdown -source: wordpress -tags: -- bootstrap -- dead tree -- publishing -title: 'WIP: Dead Tree Publishing 4' -updated: 2015-04-30 18:25:40-07:00 -wordpress_id: 214 -wordpress_slug: wip-dead-tree-publishing-4 ---- -I consider Dead Tree Publishing to be **G**ood **E**nough at this point. Itâs launched. - -[![alt:2015-04-30-180400_1366x768](https://blog.za3k.com/wp-content/uploads/2015/04/2015-04-30-180400_1366x768.jpg)](https://blog.za3k.com/wp-content/uploads/2015/04/2015-04-30-180400_1366x768.jpg) - -New site style, FAQ page - -Iâm going to add support for URLs instead of uploading PDFs, and fix some bugs here for there, but itâs essentially done. - -Meanwhile, Iâve already received my first physical book Iâm publishing through the site. I got this nice email from Eric Eve: - -``` -Dear Zachary, - -Thank you for your interest in my work. Yes, do please feel free to offer print copies of the TADS 3 Tour Guide (or any other of my TADS 3 documentation) along the lines you suggest. The only copyright right Iâm interested in enforcing is my right to be identified as the author of the work, and I'm sure that's not as issue here. - -Best wishes, - -Eric -``` diff --git a/posts-md/wip-dead-tree-publishing.md b/posts-md/wip-dead-tree-publishing.md deleted file mode 100644 index 5650601..0000000 --- a/posts-md/wip-dead-tree-publishing.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -author: admin -categories: -- Non-Technical -date: 2015-03-29 14:12:31-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- dead tree -- digital -- PDF -- physical -- publishing -- website -title: 'WIP: Dead Tree Publishing' -updated: 2015-04-28 23:09:22-07:00 -wordpress_id: 147 -wordpress_slug: wip-dead-tree-publishing ---- -I started work on my publishing website again (Dead Tree Publishing). The idea is to make a really, really convenient way to get a physical copy of a PDF/epub book. Think: âsend me a printed copy of this mailing list / tumblrâ. Right now things are looking encouraging. - -I use a âback endâ publisher who does all the actual printing, and the one I was using before charged quite a lot and wasnât amazingly fast; I just used them because they were the only publisher who was at all up to date. Seriously, order of $100 â $200 for a 100 page book, just absolutely ridiculous levels of expensive.Â Iâm switching over to a new publisher who can offer that same book for something like $7 (maybe $12 in hardback), which is absolutely reasonable, and with similar 2-week turnaround times. - -First you upload a PDF: - -[![alt:2](https://blog.za3k.com/wp-content/uploads/2015/03/2-300x90.jpg)](https://blog.za3k.com/wp-content/uploads/2015/03/1.jpg) - -Uploading a book - -Then I tell you what your ordering options are (hardcover, softcover, color), and what they cost. Iâm also supposed to ask you your address to ship the book, and for you to pay for it, Â but those arenât done yet. - -[![alt:1](https://blog.za3k.com/wp-content/uploads/2015/03/1-300x254.jpg)](https://blog.za3k.com/wp-content/uploads/2015/03/1.jpg) - -Book-buying options - -Hopefully in the next day or two Iâll have something up and running so people can order books, and then make it gradually nicer! Iâm very excited about this website existing. diff --git a/posts-md/xp-boot-usb-stick.md b/posts-md/xp-boot-usb-stick.md deleted file mode 100644 index 1531299..0000000 --- a/posts-md/xp-boot-usb-stick.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -author: admin -categories: -- Technical -date: 2015-04-13 17:46:31-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- boot -- installer -- iso -- os -- system administration -- windows -- windows xp -title: XP Boot USB Stick -updated: 2015-04-13 17:51:06-07:00 -wordpress_id: 158 -wordpress_slug: xp-boot-usb-stick ---- -Most of the following taken from :Â [http://www.msfn.org/board/topic/151992-install-xp-from-usb-without-extra-tools/](http://www.msfn.org/board/topic/151992-install-xp-from-usb-without-extra-tools/), just modified to include syslinux support. - -Let me know if there are any omissions; it an XP installer bluescreens on boot for me so I canât actually test. - -1. Obtain an XP iso file -2. Format drive with one FAT parition, marked bootable. -3. ``` - syslinux -i /dev/sdXX - ``` - -4. ``` - $ cp /usr/lib/syslinux/bios/mbr.bin >/dev/sdX - ``` - -5. ``` - $ mount /dev/sdXX /mnt - ``` - -6. ``` - mkdir /tmp/xp_iso - mount xp.iso /tmp/xp_iso - cp -ar /tmp/xp_iso/* /mnt - umount /tmp/xp_iso - rmdir xp_iso - ``` - -7. ``` - cp /usr/lib/syslinux/bios/{chain.c32,libutil.c32,menu.c32,libcom.c32} /mnt - ``` - -8. ``` - cp /mnt/I386/{NTDETECT.COM,SETUPLDR.BIN,TXTSETUP.SIF} /mnt - ``` - -9. Edit /mnt/syslinux.cfg: - - ``` - UI menu.c32# Windows XP - LABEL windows_xp - MENU LABEL Run Windows ^XP Setup - COM32 chain.c32 - APPEND fs ntldr=SETUPLDR.BIN - ``` - -10. ``` - umount /mnt - ``` - -11. Boot from the USB stick diff --git a/posts-md/zorchpad-keyboard-update.md b/posts-md/zorchpad-keyboard-update.md deleted file mode 100644 index ce21a54..0000000 --- a/posts-md/zorchpad-keyboard-update.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -author: admin -categories: -- Technical -date: 2024-05-26 18:25:37-07:00 -has-comments: false -markup: markdown -source: wordpress -tags: -- hardware -- zorchpad -title: Zorchpad keyboard update -updated: 2024-05-26 18:28:14-07:00 -wordpress_id: 1372 -wordpress_slug: zorchpad-keyboard-update ---- -The Zorchpad needs a custom keyboard. Its power budget is only 1mW, and thereâs just nothing available in that range. So, I need to make a custom keyboard. I started reading up on how to make your ownâespecially the electronics. - -I donât know how to make a PCB: - -![](https://blog.za3k.com/wp-content/uploads/2024/05/image-1.png) - -PCB from HacKeyboard - -Or how to attach headers to the inside of an existing keyboard, which looks like thisâ: - -![alt:Mapping the Innards of a Keyboard : 7 Steps (with Pictures) - Instructables](https://content.instructables.com/FOM/VMTN/HZN0VZGV/FOMVMTNHZN0VZGV.jpg?auto=webp&frame=1&width=2100) - -But I found a project called [GOLEM](https://golem.hu/guide/keyboard-build-logs/) with an excellent guide to making your own keyboard. Here is their wiring: - -![](https://blog.za3k.com/wp-content/uploads/2024/05/image.png) - -GOLEM Macropad - -I can do that! They got me out of a major rut. - ---- - -[Their advice](https://golem.hu/guide/first-macropad) walks you through how to do a small keyboard in a cardboard plate. I did a few keys, gauged the effort, and decided to use my 3D printer. Cutting out 50-60 keys precisely by hand doesnât sound easy. Worse, if you mess up, you have to start over. In plastic, I canât mess up halfway, and the spacers to support the keyboard can be part of the print. - -[![](https://blog.za3k.com/wp-content/uploads/2024/05/2024-05-20-233106_2560x1440_scrot-1024x576.png)](https://blog.za3k.com/wp-content/uploads/2024/05/2024-05-20-233106_2560x1440_scrot.png) - -Above, Iâm designing a âsamplerâ keyboard in CAD (OpenSCAD). I want to iron out problems in my process before I try a full-size keyboard. Below, Prusa-Slic3r is slicing the finished model for my 3D printer to print. - -[![](https://blog.za3k.com/wp-content/uploads/2024/05/2024-05-20-235849_1920x1080_scrot-1024x576.png)](https://blog.za3k.com/wp-content/uploads/2024/05/2024-05-20-235849_1920x1080_scrot.png) - -Hereâs the finished sampler keyboard: - -[![](https://blog.za3k.com/wp-content/uploads/2024/05/tiny_keyboard2-1-1024x747.jpg)](https://blog.za3k.com/wp-content/uploads/2024/05/tiny_keyboard2-1.jpg) - -Currently Iâm waiting on keycaps and switches ordered from China, and then Iâll put together my finished keyboard. But I have been making some progress in the meantime. Hereâs the layout Iâm going to try. - -[![](https://blog.za3k.com/wp-content/uploads/2024/05/keyboard57.png)](https://blog.za3k.com/wp-content/uploads/2024/05/keyboard57.png) - -And Iâve started streaming some development of a case and keyboard on [Twitch](https://www.twitch.tv/za3k) (Tue/Thu 12pm noon, EDT). Feel free to join! Anyone can watch, but you need an account to chat. - -[![](https://blog.za3k.com/wp-content/uploads/2024/05/stream-1024x576.png)](https://blog.za3k.com/wp-content/uploads/2024/05/stream.png) diff --git a/posts-md/116.md b/posts/116.md similarity index 100% rename from posts-md/116.md rename to posts/116.md diff --git a/posts-md/20-minute-interruptions.md b/posts/20-minute-interruptions.md similarity index 100% rename from posts-md/20-minute-interruptions.md rename to posts/20-minute-interruptions.md diff --git a/posts-md/2020-books.md b/posts/2020-books.md similarity index 100% rename from posts-md/2020-books.md rename to posts/2020-books.md diff --git a/posts-md/2020-review.md b/posts/2020-review.md similarity index 100% rename from posts-md/2020-review.md rename to posts/2020-review.md diff --git a/posts-md/2020-videogames.md b/posts/2020-videogames.md similarity index 100% rename from posts-md/2020-videogames.md rename to posts/2020-videogames.md diff --git a/posts-md/2021-books.md b/posts/2021-books.md similarity index 100% rename from posts-md/2021-books.md rename to posts/2021-books.md diff --git a/posts-md/2022-books.md b/posts/2022-books.md similarity index 100% rename from posts-md/2022-books.md rename to posts/2022-books.md diff --git a/posts-md/2022-year-in-review.md b/posts/2022-year-in-review.md similarity index 100% rename from posts-md/2022-year-in-review.md rename to posts/2022-year-in-review.md diff --git a/posts-md/2023-flash-media-longevity-testing-3-years-later.md b/posts/2023-flash-media-longevity-testing-3-years-later.md similarity index 100% rename from posts-md/2023-flash-media-longevity-testing-3-years-later.md rename to posts/2023-flash-media-longevity-testing-3-years-later.md diff --git a/posts-md/3-more-games.md b/posts/3-more-games.md similarity index 100% rename from posts-md/3-more-games.md rename to posts/3-more-games.md diff --git a/posts-md/3-new-games-deadly-education-rpg-logic-potions-emperical-zendo.md b/posts/3-new-games-deadly-education-rpg-logic-potions-emperical-zendo.md similarity index 100% rename from posts-md/3-new-games-deadly-education-rpg-logic-potions-emperical-zendo.md rename to posts/3-new-games-deadly-education-rpg-logic-potions-emperical-zendo.md diff --git a/posts-md/30-days-of-learning-play-and-newness.md b/posts/30-days-of-learning-play-and-newness.md similarity index 100% rename from posts-md/30-days-of-learning-play-and-newness.md rename to posts/30-days-of-learning-play-and-newness.md diff --git a/posts-md/known-good/a-mystery-in-the-text-editor.md b/posts/a-mystery-in-the-text-editor.md similarity index 100% rename from posts-md/known-good/a-mystery-in-the-text-editor.md rename to posts/a-mystery-in-the-text-editor.md diff --git a/posts-md/a-pixel-art.md b/posts/a-pixel-art.md similarity index 100% rename from posts-md/a-pixel-art.md rename to posts/a-pixel-art.md diff --git a/posts-md/all-the-recipes.md b/posts/all-the-recipes.md similarity index 100% rename from posts-md/all-the-recipes.md rename to posts/all-the-recipes.md diff --git a/posts-md/amazon-aws.md b/posts/amazon-aws.md similarity index 100% rename from posts-md/amazon-aws.md rename to posts/amazon-aws.md diff --git a/posts-md/android-backup-on-arch-linux.md b/posts/android-backup-on-arch-linux.md similarity index 100% rename from posts-md/android-backup-on-arch-linux.md rename to posts/android-backup-on-arch-linux.md diff --git a/posts-md/april-fools-puzzle-contest-solutions.md b/posts/april-fools-puzzle-contest-solutions.md similarity index 100% rename from posts-md/april-fools-puzzle-contest-solutions.md rename to posts/april-fools-puzzle-contest-solutions.md diff --git a/posts-md/april-fools-puzzle-contest.md b/posts/april-fools-puzzle-contest.md similarity index 100% rename from posts-md/april-fools-puzzle-contest.md rename to posts/april-fools-puzzle-contest.md diff --git a/posts-md/archiving-all-bash-commands-typed.md b/posts/archiving-all-bash-commands-typed.md similarity index 100% rename from posts-md/archiving-all-bash-commands-typed.md rename to posts/archiving-all-bash-commands-typed.md diff --git a/posts-md/archiving-all-web-traffic.md b/posts/archiving-all-web-traffic.md similarity index 100% rename from posts-md/archiving-all-web-traffic.md rename to posts/archiving-all-web-traffic.md diff --git a/posts-md/archiving-github.md b/posts/archiving-github.md similarity index 100% rename from posts-md/archiving-github.md rename to posts/archiving-github.md diff --git a/posts-md/archiving-gmail.md b/posts/archiving-gmail.md similarity index 100% rename from posts-md/archiving-gmail.md rename to posts/archiving-gmail.md diff --git a/posts-md/archiving-twitch.md b/posts/archiving-twitch.md similarity index 100% rename from posts-md/archiving-twitch.md rename to posts/archiving-twitch.md diff --git a/posts-md/known-good/archiving-twitter.md b/posts/archiving-twitter.md similarity index 100% rename from posts-md/known-good/archiving-twitter.md rename to posts/archiving-twitter.md diff --git a/posts-md/articles-section.md b/posts/articles-section.md similarity index 100% rename from posts-md/articles-section.md rename to posts/articles-section.md diff --git a/posts-md/backup-android-on-plugin.md b/posts/backup-android-on-plugin.md similarity index 100% rename from posts-md/backup-android-on-plugin.md rename to posts/backup-android-on-plugin.md diff --git a/posts-md/banh-chung.md b/posts/banh-chung.md similarity index 100% rename from posts-md/banh-chung.md rename to posts/banh-chung.md diff --git a/posts-md/known-good/blast-furance.md b/posts/blast-furance.md similarity index 100% rename from posts-md/known-good/blast-furance.md rename to posts/blast-furance.md diff --git a/posts-md/blueprint-maker-13-complete.md b/posts/blueprint-maker-13-complete.md similarity index 100% rename from posts-md/blueprint-maker-13-complete.md rename to posts/blueprint-maker-13-complete.md diff --git a/posts-md/board-game-travel-kit.md b/posts/board-game-travel-kit.md similarity index 100% rename from posts-md/board-game-travel-kit.md rename to posts/board-game-travel-kit.md diff --git a/posts-md/capturing-video-on-debian-linux-with-the-blackmagic-intensity-pro-4k-card.md b/posts/capturing-video-on-debian-linux-with-the-blackmagic-intensity-pro-4k-card.md similarity index 100% rename from posts-md/capturing-video-on-debian-linux-with-the-blackmagic-intensity-pro-4k-card.md rename to posts/capturing-video-on-debian-linux-with-the-blackmagic-intensity-pro-4k-card.md diff --git a/posts-md/cardboard-mail-holders.md b/posts/cardboard-mail-holders.md similarity index 100% rename from posts-md/cardboard-mail-holders.md rename to posts/cardboard-mail-holders.md diff --git a/posts-md/configuring-mailxs-mailrc-with-gmail.md b/posts/configuring-mailxs-mailrc-with-gmail.md similarity index 100% rename from posts-md/configuring-mailxs-mailrc-with-gmail.md rename to posts/configuring-mailxs-mailrc-with-gmail.md diff --git a/posts-md/controlling-a-computercraft-turtle-remotely.md b/posts/controlling-a-computercraft-turtle-remotely.md similarity index 100% rename from posts-md/controlling-a-computercraft-turtle-remotely.md rename to posts/controlling-a-computercraft-turtle-remotely.md diff --git a/posts-md/cookbook.md b/posts/cookbook.md similarity index 100% rename from posts-md/cookbook.md rename to posts/cookbook.md diff --git a/posts-md/crawling-etiquette.md b/posts/crawling-etiquette.md similarity index 100% rename from posts-md/crawling-etiquette.md rename to posts/crawling-etiquette.md diff --git a/posts-md/cron-email-and-sending-email-to-only-one-address.md b/posts/cron-email-and-sending-email-to-only-one-address.md similarity index 100% rename from posts-md/cron-email-and-sending-email-to-only-one-address.md rename to posts/cron-email-and-sending-email-to-only-one-address.md diff --git a/posts-md/dd-spells-srd-vs-5e-players-handbook.md b/posts/dd-spells-srd-vs-5e-players-handbook.md similarity index 100% rename from posts-md/dd-spells-srd-vs-5e-players-handbook.md rename to posts/dd-spells-srd-vs-5e-players-handbook.md diff --git a/posts-md/dd-story-time.md b/posts/dd-story-time.md similarity index 100% rename from posts-md/dd-story-time.md rename to posts/dd-story-time.md diff --git a/posts-md/ddos.md b/posts/ddos.md similarity index 100% rename from posts-md/ddos.md rename to posts/ddos.md diff --git a/posts-md/known-good/default-twitter-icons.md b/posts/default-twitter-icons.md similarity index 100% rename from posts-md/known-good/default-twitter-icons.md rename to posts/default-twitter-icons.md diff --git a/posts-md/dependency-resolution-in-javascript.md b/posts/dependency-resolution-in-javascript.md similarity index 100% rename from posts-md/dependency-resolution-in-javascript.md rename to posts/dependency-resolution-in-javascript.md diff --git a/posts-md/diy-hard-drive-carrying-case.md b/posts/diy-hard-drive-carrying-case.md similarity index 100% rename from posts-md/diy-hard-drive-carrying-case.md rename to posts/diy-hard-drive-carrying-case.md diff --git a/posts-md/known-good/diy-keyboards-and-how-keyboards-work.md b/posts/diy-keyboards-and-how-keyboards-work.md similarity index 100% rename from posts-md/known-good/diy-keyboards-and-how-keyboards-work.md rename to posts/diy-keyboards-and-how-keyboards-work.md diff --git a/posts-md/domain-names-for-sale.md b/posts/domain-names-for-sale.md similarity index 100% rename from posts-md/domain-names-for-sale.md rename to posts/domain-names-for-sale.md diff --git a/posts-md/dungeon-master-ii-spell-runes.md b/posts/dungeon-master-ii-spell-runes.md similarity index 100% rename from posts-md/dungeon-master-ii-spell-runes.md rename to posts/dungeon-master-ii-spell-runes.md diff --git a/posts-md/e-ink-laptop.md b/posts/e-ink-laptop.md similarity index 100% rename from posts-md/e-ink-laptop.md rename to posts/e-ink-laptop.md diff --git a/posts-md/easel-toy.md b/posts/easel-toy.md similarity index 100% rename from posts-md/easel-toy.md rename to posts/easel-toy.md diff --git a/posts-md/encrypted-root-on-debian-part-2-unattended-boot.md b/posts/encrypted-root-on-debian-part-2-unattended-boot.md similarity index 100% rename from posts-md/encrypted-root-on-debian-part-2-unattended-boot.md rename to posts/encrypted-root-on-debian-part-2-unattended-boot.md diff --git a/posts-md/etherpad.md b/posts/etherpad.md similarity index 100% rename from posts-md/etherpad.md rename to posts/etherpad.md diff --git a/posts-md/even-more-uri-handlers-in-linux.md b/posts/even-more-uri-handlers-in-linux.md similarity index 100% rename from posts-md/even-more-uri-handlers-in-linux.md rename to posts/even-more-uri-handlers-in-linux.md diff --git a/posts-md/fabric1-aur-package.md b/posts/fabric1-aur-package.md similarity index 100% rename from posts-md/fabric1-aur-package.md rename to posts/fabric1-aur-package.md diff --git a/posts-md/first-aid-kit.md b/posts/first-aid-kit.md similarity index 100% rename from posts-md/first-aid-kit.md rename to posts/first-aid-kit.md diff --git a/posts-md/flash-media-longevity-testing-4-years-later.md b/posts/flash-media-longevity-testing-4-years-later.md similarity index 100% rename from posts-md/flash-media-longevity-testing-4-years-later.md rename to posts/flash-media-longevity-testing-4-years-later.md diff --git a/posts-md/games-i-like.md b/posts/games-i-like.md similarity index 100% rename from posts-md/games-i-like.md rename to posts/games-i-like.md diff --git a/posts-md/garden-signs-on-wall-tiles-pt-2.md b/posts/garden-signs-on-wall-tiles-pt-2.md similarity index 100% rename from posts-md/garden-signs-on-wall-tiles-pt-2.md rename to posts/garden-signs-on-wall-tiles-pt-2.md diff --git a/posts-md/garden-signs-on-wall-tiles.md b/posts/garden-signs-on-wall-tiles.md similarity index 100% rename from posts-md/garden-signs-on-wall-tiles.md rename to posts/garden-signs-on-wall-tiles.md diff --git a/posts-md/getting-rid-of-mold.md b/posts/getting-rid-of-mold.md similarity index 100% rename from posts-md/getting-rid-of-mold.md rename to posts/getting-rid-of-mold.md diff --git a/posts-md/getting-the-adafruit-pro-trinket-3-3v-to-work-in-arch-linux.md b/posts/getting-the-adafruit-pro-trinket-3-3v-to-work-in-arch-linux.md similarity index 100% rename from posts-md/getting-the-adafruit-pro-trinket-3-3v-to-work-in-arch-linux.md rename to posts/getting-the-adafruit-pro-trinket-3-3v-to-work-in-arch-linux.md diff --git a/posts-md/github-com-archive-background-research.md b/posts/github-com-archive-background-research.md similarity index 100% rename from posts-md/github-com-archive-background-research.md rename to posts/github-com-archive-background-research.md diff --git a/posts-md/good-time-estimation.md b/posts/good-time-estimation.md similarity index 100% rename from posts-md/good-time-estimation.md rename to posts/good-time-estimation.md diff --git a/posts-md/hack-a-day-2023-2.md b/posts/hack-a-day-2023-2.md similarity index 100% rename from posts-md/hack-a-day-2023-2.md rename to posts/hack-a-day-2023-2.md diff --git a/posts-md/hack-a-day-2023-is-done.md b/posts/hack-a-day-2023-is-done.md similarity index 100% rename from posts-md/hack-a-day-2023-is-done.md rename to posts/hack-a-day-2023-is-done.md diff --git a/posts-md/hack-a-day-2023.md b/posts/hack-a-day-2023.md similarity index 100% rename from posts-md/hack-a-day-2023.md rename to posts/hack-a-day-2023.md diff --git a/posts-md/hack-a-day-day-01-perquackey.md b/posts/hack-a-day-day-01-perquackey.md similarity index 100% rename from posts-md/hack-a-day-day-01-perquackey.md rename to posts/hack-a-day-day-01-perquackey.md diff --git a/posts-md/hack-a-day-day-04-lashed-table.md b/posts/hack-a-day-day-04-lashed-table.md similarity index 100% rename from posts-md/hack-a-day-day-04-lashed-table.md rename to posts/hack-a-day-day-04-lashed-table.md diff --git a/posts-md/hack-a-day-day-06-doodlemoji-alchemy.md b/posts/hack-a-day-day-06-doodlemoji-alchemy.md similarity index 100% rename from posts-md/hack-a-day-day-06-doodlemoji-alchemy.md rename to posts/hack-a-day-day-06-doodlemoji-alchemy.md diff --git a/posts-md/known-good/hack-a-day-day-06.md b/posts/hack-a-day-day-06.md similarity index 100% rename from posts-md/known-good/hack-a-day-day-06.md rename to posts/hack-a-day-day-06.md diff --git a/posts-md/hack-a-day-day-08-receipt-zine.md b/posts/hack-a-day-day-08-receipt-zine.md similarity index 100% rename from posts-md/hack-a-day-day-08-receipt-zine.md rename to posts/hack-a-day-day-08-receipt-zine.md diff --git a/posts-md/hack-a-day-day-10-typewriter.md b/posts/hack-a-day-day-10-typewriter.md similarity index 100% rename from posts-md/hack-a-day-day-10-typewriter.md rename to posts/hack-a-day-day-10-typewriter.md diff --git a/posts-md/hack-a-day-day-11-raytraced-rain.md b/posts/hack-a-day-day-11-raytraced-rain.md similarity index 100% rename from posts-md/hack-a-day-day-11-raytraced-rain.md rename to posts/hack-a-day-day-11-raytraced-rain.md diff --git a/posts-md/hack-a-day-day-12-screensavers.md b/posts/hack-a-day-day-12-screensavers.md similarity index 100% rename from posts-md/hack-a-day-day-12-screensavers.md rename to posts/hack-a-day-day-12-screensavers.md diff --git a/posts-md/hack-a-day-day-13-blueprint-maker.md b/posts/hack-a-day-day-13-blueprint-maker.md similarity index 100% rename from posts-md/hack-a-day-day-13-blueprint-maker.md rename to posts/hack-a-day-day-13-blueprint-maker.md diff --git a/posts-md/hack-a-day-day-14-bytebeat-synth.md b/posts/hack-a-day-day-14-bytebeat-synth.md similarity index 100% rename from posts-md/hack-a-day-day-14-bytebeat-synth.md rename to posts/hack-a-day-day-14-bytebeat-synth.md diff --git a/posts-md/hack-a-day-day-17-tower-of-choices.md b/posts/hack-a-day-day-17-tower-of-choices.md similarity index 100% rename from posts-md/hack-a-day-day-17-tower-of-choices.md rename to posts/hack-a-day-day-17-tower-of-choices.md diff --git a/posts-md/hack-a-day-day-18-a-i-grab-bag.md b/posts/hack-a-day-day-18-a-i-grab-bag.md similarity index 100% rename from posts-md/hack-a-day-day-18-a-i-grab-bag.md rename to posts/hack-a-day-day-18-a-i-grab-bag.md diff --git a/posts-md/hack-a-day-day-2-raytracing.md b/posts/hack-a-day-day-2-raytracing.md similarity index 100% rename from posts-md/hack-a-day-day-2-raytracing.md rename to posts/hack-a-day-day-2-raytracing.md diff --git a/posts-md/hack-a-day-day-20-hillsfar-lockpicking-spritesheet.md b/posts/hack-a-day-day-20-hillsfar-lockpicking-spritesheet.md similarity index 100% rename from posts-md/hack-a-day-day-20-hillsfar-lockpicking-spritesheet.md rename to posts/hack-a-day-day-20-hillsfar-lockpicking-spritesheet.md diff --git a/posts-md/hack-a-day-day-22-homemade-pcbs.md b/posts/hack-a-day-day-22-homemade-pcbs.md similarity index 100% rename from posts-md/hack-a-day-day-22-homemade-pcbs.md rename to posts/hack-a-day-day-22-homemade-pcbs.md diff --git a/posts-md/hack-a-day-day-23-packing.md b/posts/hack-a-day-day-23-packing.md similarity index 100% rename from posts-md/hack-a-day-day-23-packing.md rename to posts/hack-a-day-day-23-packing.md diff --git a/posts-md/hack-a-day-day-28-90s-sitcom.md b/posts/hack-a-day-day-28-90s-sitcom.md similarity index 100% rename from posts-md/hack-a-day-day-28-90s-sitcom.md rename to posts/hack-a-day-day-28-90s-sitcom.md diff --git a/posts-md/hack-a-day-day-29-speed-reading.md b/posts/hack-a-day-day-29-speed-reading.md similarity index 100% rename from posts-md/hack-a-day-day-29-speed-reading.md rename to posts/hack-a-day-day-29-speed-reading.md diff --git a/posts-md/hack-a-day-day-30-music-of-the-celestial-spheres.md b/posts/hack-a-day-day-30-music-of-the-celestial-spheres.md similarity index 100% rename from posts-md/hack-a-day-day-30-music-of-the-celestial-spheres.md rename to posts/hack-a-day-day-30-music-of-the-celestial-spheres.md diff --git a/posts-md/hack-a-day-hack-a-battle.md b/posts/hack-a-day-hack-a-battle.md similarity index 100% rename from posts-md/hack-a-day-hack-a-battle.md rename to posts/hack-a-day-hack-a-battle.md diff --git a/posts-md/hack-a-day-hack-a-blog.md b/posts/hack-a-day-hack-a-blog.md similarity index 100% rename from posts-md/hack-a-day-hack-a-blog.md rename to posts/hack-a-day-hack-a-blog.md diff --git a/posts-md/hack-a-day-hack-a-bug.md b/posts/hack-a-day-hack-a-bug.md similarity index 100% rename from posts-md/hack-a-day-hack-a-bug.md rename to posts/hack-a-day-hack-a-bug.md diff --git a/posts-md/hack-a-day-hack-a-chat.md b/posts/hack-a-day-hack-a-chat.md similarity index 100% rename from posts-md/hack-a-day-hack-a-chat.md rename to posts/hack-a-day-hack-a-chat.md diff --git a/posts-md/known-good/hack-a-day-hack-a-clock.md b/posts/hack-a-day-hack-a-clock.md similarity index 100% rename from posts-md/known-good/hack-a-day-hack-a-clock.md rename to posts/hack-a-day-hack-a-clock.md diff --git a/posts-md/hack-a-day-hack-a-crop.md b/posts/hack-a-day-hack-a-crop.md similarity index 100% rename from posts-md/hack-a-day-hack-a-crop.md rename to posts/hack-a-day-hack-a-crop.md diff --git a/posts-md/hack-a-day-hack-a-dictionary.md b/posts/hack-a-day-hack-a-dictionary.md similarity index 100% rename from posts-md/hack-a-day-hack-a-dictionary.md rename to posts/hack-a-day-hack-a-dictionary.md diff --git a/posts-md/hack-a-day-hack-a-farm.md b/posts/hack-a-day-hack-a-farm.md similarity index 100% rename from posts-md/hack-a-day-hack-a-farm.md rename to posts/hack-a-day-hack-a-farm.md diff --git a/posts-md/known-good/hack-a-day-hack-a-hang.md b/posts/hack-a-day-hack-a-hang.md similarity index 100% rename from posts-md/known-good/hack-a-day-hack-a-hang.md rename to posts/hack-a-day-hack-a-hang.md diff --git a/posts-md/hack-a-day-hack-a-hell.md b/posts/hack-a-day-hack-a-hell.md similarity index 100% rename from posts-md/hack-a-day-hack-a-hell.md rename to posts/hack-a-day-hack-a-hell.md diff --git a/posts-md/hack-a-day-hack-a-homepage.md b/posts/hack-a-day-hack-a-homepage.md similarity index 100% rename from posts-md/hack-a-day-hack-a-homepage.md rename to posts/hack-a-day-hack-a-homepage.md diff --git a/posts-md/hack-a-day-hack-a-line.md b/posts/hack-a-day-hack-a-line.md similarity index 100% rename from posts-md/hack-a-day-hack-a-line.md rename to posts/hack-a-day-hack-a-line.md diff --git a/posts-md/hack-a-day-hack-a-link-2.md b/posts/hack-a-day-hack-a-link-2.md similarity index 100% rename from posts-md/hack-a-day-hack-a-link-2.md rename to posts/hack-a-day-hack-a-link-2.md diff --git a/posts-md/hack-a-day-hack-a-link.md b/posts/hack-a-day-hack-a-link.md similarity index 100% rename from posts-md/hack-a-day-hack-a-link.md rename to posts/hack-a-day-hack-a-link.md diff --git a/posts-md/hack-a-day-hack-a-machine.md b/posts/hack-a-day-hack-a-machine.md similarity index 100% rename from posts-md/hack-a-day-hack-a-machine.md rename to posts/hack-a-day-hack-a-machine.md diff --git a/posts-md/hack-a-day-hack-a-mandelbrot.md b/posts/hack-a-day-hack-a-mandelbrot.md similarity index 100% rename from posts-md/hack-a-day-hack-a-mandelbrot.md rename to posts/hack-a-day-hack-a-mandelbrot.md diff --git a/posts-md/hack-a-day-hack-a-minigame.md b/posts/hack-a-day-hack-a-minigame.md similarity index 100% rename from posts-md/hack-a-day-hack-a-minigame.md rename to posts/hack-a-day-hack-a-minigame.md diff --git a/posts-md/hack-a-day-hack-a-paste.md b/posts/hack-a-day-hack-a-paste.md similarity index 100% rename from posts-md/hack-a-day-hack-a-paste.md rename to posts/hack-a-day-hack-a-paste.md diff --git a/posts-md/hack-a-day-hack-a-snake.md b/posts/hack-a-day-hack-a-snake.md similarity index 100% rename from posts-md/hack-a-day-hack-a-snake.md rename to posts/hack-a-day-hack-a-snake.md diff --git a/posts-md/hack-a-day-hack-a-song.md b/posts/hack-a-day-hack-a-song.md similarity index 100% rename from posts-md/hack-a-day-hack-a-song.md rename to posts/hack-a-day-hack-a-song.md diff --git a/posts-md/hack-a-day-hack-a-sound.md b/posts/hack-a-day-hack-a-sound.md similarity index 100% rename from posts-md/hack-a-day-hack-a-sound.md rename to posts/hack-a-day-hack-a-sound.md diff --git a/posts-md/hack-a-day-hack-a-stats.md b/posts/hack-a-day-hack-a-stats.md similarity index 100% rename from posts-md/hack-a-day-hack-a-stats.md rename to posts/hack-a-day-hack-a-stats.md diff --git a/posts-md/hack-a-day-hack-a-tank.md b/posts/hack-a-day-hack-a-tank.md similarity index 100% rename from posts-md/hack-a-day-hack-a-tank.md rename to posts/hack-a-day-hack-a-tank.md diff --git a/posts-md/hack-a-day-hack-a-tile.md b/posts/hack-a-day-hack-a-tile.md similarity index 100% rename from posts-md/hack-a-day-hack-a-tile.md rename to posts/hack-a-day-hack-a-tile.md diff --git a/posts-md/hack-a-day-hack-a-tv-guide.md b/posts/hack-a-day-hack-a-tv-guide.md similarity index 100% rename from posts-md/hack-a-day-hack-a-tv-guide.md rename to posts/hack-a-day-hack-a-tv-guide.md diff --git a/posts-md/hack-a-day-hack-an-adventure.md b/posts/hack-a-day-hack-an-adventure.md similarity index 100% rename from posts-md/hack-a-day-hack-an-adventure.md rename to posts/hack-a-day-hack-an-adventure.md diff --git a/posts-md/hack-a-day-hack-an-asteroid.md b/posts/hack-a-day-hack-an-asteroid.md similarity index 100% rename from posts-md/hack-a-day-hack-an-asteroid.md rename to posts/hack-a-day-hack-an-asteroid.md diff --git a/posts-md/hack-a-day-hack-an-experiment.md b/posts/hack-a-day-hack-an-experiment.md similarity index 100% rename from posts-md/hack-a-day-hack-an-experiment.md rename to posts/hack-a-day-hack-an-experiment.md diff --git a/posts-md/hack-a-day-hack-an-icecube-failure.md b/posts/hack-a-day-hack-an-icecube-failure.md similarity index 100% rename from posts-md/hack-a-day-hack-an-icecube-failure.md rename to posts/hack-a-day-hack-an-icecube-failure.md diff --git a/posts-md/hack-a-day-hack-an-mmo.md b/posts/hack-a-day-hack-an-mmo.md similarity index 100% rename from posts-md/hack-a-day-hack-an-mmo.md rename to posts/hack-a-day-hack-an-mmo.md diff --git a/posts-md/hack-a-day-hack-an-uptime.md b/posts/hack-a-day-hack-an-uptime.md similarity index 100% rename from posts-md/hack-a-day-hack-an-uptime.md rename to posts/hack-a-day-hack-an-uptime.md diff --git a/posts-md/hack-a-day-website.md b/posts/hack-a-day-website.md similarity index 100% rename from posts-md/hack-a-day-website.md rename to posts/hack-a-day-website.md diff --git a/posts-md/hello-world.md b/posts/hello-world.md similarity index 100% rename from posts-md/hello-world.md rename to posts/hello-world.md diff --git a/posts-md/hillsfar-lockpicking-20-complete.md b/posts/hillsfar-lockpicking-20-complete.md similarity index 100% rename from posts-md/hillsfar-lockpicking-20-complete.md rename to posts/hillsfar-lockpicking-20-complete.md diff --git a/posts-md/how-to-retire-for-infinity-years.md b/posts/how-to-retire-for-infinity-years.md similarity index 100% rename from posts-md/how-to-retire-for-infinity-years.md rename to posts/how-to-retire-for-infinity-years.md diff --git a/posts-md/html-css-cheatsheet.md b/posts/html-css-cheatsheet.md similarity index 100% rename from posts-md/html-css-cheatsheet.md rename to posts/html-css-cheatsheet.md diff --git a/posts-md/installing-canon-imageclass-lbp-6000-on-64-bit-debian.md b/posts/installing-canon-imageclass-lbp-6000-on-64-bit-debian.md similarity index 100% rename from posts-md/installing-canon-imageclass-lbp-6000-on-64-bit-debian.md rename to posts/installing-canon-imageclass-lbp-6000-on-64-bit-debian.md diff --git a/posts-md/installing-email-with-postfix-and-dovecot.md b/posts/installing-email-with-postfix-and-dovecot.md similarity index 100% rename from posts-md/installing-email-with-postfix-and-dovecot.md rename to posts/installing-email-with-postfix-and-dovecot.md diff --git a/posts-md/introducing-the-zorchpad-display-demo.md b/posts/introducing-the-zorchpad-display-demo.md similarity index 100% rename from posts-md/introducing-the-zorchpad-display-demo.md rename to posts/introducing-the-zorchpad-display-demo.md diff --git a/posts-md/irc.md b/posts/irc.md similarity index 100% rename from posts-md/irc.md rename to posts/irc.md diff --git a/posts-md/ircpuzzles-2024.md b/posts/ircpuzzles-2024.md similarity index 100% rename from posts-md/ircpuzzles-2024.md rename to posts/ircpuzzles-2024.md diff --git a/posts-md/is-rick-and-morty-out-season-5.md b/posts/is-rick-and-morty-out-season-5.md similarity index 100% rename from posts-md/is-rick-and-morty-out-season-5.md rename to posts/is-rick-and-morty-out-season-5.md diff --git a/posts-md/is-rick-and-morty-out-season-6.md b/posts/is-rick-and-morty-out-season-6.md similarity index 100% rename from posts-md/is-rick-and-morty-out-season-6.md rename to posts/is-rick-and-morty-out-season-6.md diff --git a/posts-md/life-logging-in-2019.md b/posts/life-logging-in-2019.md similarity index 100% rename from posts-md/life-logging-in-2019.md rename to posts/life-logging-in-2019.md diff --git a/posts-md/linux-print-server.md b/posts/linux-print-server.md similarity index 100% rename from posts-md/linux-print-server.md rename to posts/linux-print-server.md diff --git a/posts-md/mail-filtering-with-dovecot.md b/posts/mail-filtering-with-dovecot.md similarity index 100% rename from posts-md/mail-filtering-with-dovecot.md rename to posts/mail-filtering-with-dovecot.md diff --git a/posts-md/making-a-hardware-random-number-generator.md b/posts/making-a-hardware-random-number-generator.md similarity index 100% rename from posts-md/making-a-hardware-random-number-generator.md rename to posts/making-a-hardware-random-number-generator.md diff --git a/posts-md/making-my-finances-public.md b/posts/making-my-finances-public.md similarity index 100% rename from posts-md/making-my-finances-public.md rename to posts/making-my-finances-public.md diff --git a/posts-md/making-signs-on-wall-tiles.md b/posts/making-signs-on-wall-tiles.md similarity index 100% rename from posts-md/making-signs-on-wall-tiles.md rename to posts/making-signs-on-wall-tiles.md diff --git a/posts-md/markdown-support.md b/posts/markdown-support.md similarity index 100% rename from posts-md/markdown-support.md rename to posts/markdown-support.md diff --git a/posts-md/meeple-initiative-tracker.md b/posts/meeple-initiative-tracker.md similarity index 100% rename from posts-md/meeple-initiative-tracker.md rename to posts/meeple-initiative-tracker.md diff --git a/posts-md/known-good/migrating-an-existing-debian-installation-to-encrypted-root.md b/posts/migrating-an-existing-debian-installation-to-encrypted-root.md similarity index 100% rename from posts-md/known-good/migrating-an-existing-debian-installation-to-encrypted-root.md rename to posts/migrating-an-existing-debian-installation-to-encrypted-root.md diff --git a/posts-md/mon8.md b/posts/mon8.md similarity index 100% rename from posts-md/mon8.md rename to posts/mon8.md diff --git a/posts-md/known-good/money-orders.md b/posts/money-orders.md similarity index 100% rename from posts-md/known-good/money-orders.md rename to posts/money-orders.md diff --git a/posts-md/moreorcs-com.md b/posts/moreorcs-com.md similarity index 100% rename from posts-md/moreorcs-com.md rename to posts/moreorcs-com.md diff --git a/posts-md/multi-universe-rpg-toy.md b/posts/multi-universe-rpg-toy.md similarity index 100% rename from posts-md/multi-universe-rpg-toy.md rename to posts/multi-universe-rpg-toy.md diff --git a/posts-md/my-todo-list.md b/posts/my-todo-list.md similarity index 100% rename from posts-md/my-todo-list.md rename to posts/my-todo-list.md diff --git a/posts-md/new-experimental-blog.md b/posts/new-experimental-blog.md similarity index 100% rename from posts-md/new-experimental-blog.md rename to posts/new-experimental-blog.md diff --git a/posts-md/ogs2021-27-million-go-games.md b/posts/ogs2021-27-million-go-games.md similarity index 100% rename from posts-md/ogs2021-27-million-go-games.md rename to posts/ogs2021-27-million-go-games.md diff --git a/posts-md/ok-mixnet.md b/posts/ok-mixnet.md similarity index 100% rename from posts-md/ok-mixnet.md rename to posts/ok-mixnet.md diff --git a/posts-md/old-wikipedia-urdu.md b/posts/old-wikipedia-urdu.md similarity index 100% rename from posts-md/old-wikipedia-urdu.md rename to posts/old-wikipedia-urdu.md diff --git a/posts-md/old-wikipedia.md b/posts/old-wikipedia.md similarity index 100% rename from posts-md/old-wikipedia.md rename to posts/old-wikipedia.md diff --git a/posts-md/one-page-rpgs.md b/posts/one-page-rpgs.md similarity index 100% rename from posts-md/one-page-rpgs.md rename to posts/one-page-rpgs.md diff --git a/posts-md/one-screenshot-per-minute.md b/posts/one-screenshot-per-minute.md similarity index 100% rename from posts-md/one-screenshot-per-minute.md rename to posts/one-screenshot-per-minute.md diff --git a/posts-md/open-nntp-server.md b/posts/open-nntp-server.md similarity index 100% rename from posts-md/open-nntp-server.md rename to posts/open-nntp-server.md diff --git a/posts-md/painting.md b/posts/painting.md similarity index 100% rename from posts-md/painting.md rename to posts/painting.md diff --git a/posts-md/pandora-songs.md b/posts/pandora-songs.md similarity index 100% rename from posts-md/pandora-songs.md rename to posts/pandora-songs.md diff --git a/posts-md/paper-archival.md b/posts/paper-archival.md similarity index 100% rename from posts-md/paper-archival.md rename to posts/paper-archival.md diff --git a/posts-md/pixel-alphabet.md b/posts/pixel-alphabet.md similarity index 100% rename from posts-md/pixel-alphabet.md rename to posts/pixel-alphabet.md diff --git a/posts-md/pixel-art-a-cool-cat.md b/posts/pixel-art-a-cool-cat.md similarity index 100% rename from posts-md/pixel-art-a-cool-cat.md rename to posts/pixel-art-a-cool-cat.md diff --git a/posts-md/pompompom.md b/posts/pompompom.md similarity index 100% rename from posts-md/pompompom.md rename to posts/pompompom.md diff --git a/posts-md/known-good/postmortem-bs-store.md b/posts/postmortem-bs-store.md similarity index 100% rename from posts-md/known-good/postmortem-bs-store.md rename to posts/postmortem-bs-store.md diff --git a/posts-md/printable-todo-list.md b/posts/printable-todo-list.md similarity index 100% rename from posts-md/printable-todo-list.md rename to posts/printable-todo-list.md diff --git a/posts-md/printing-on-the-brother-hl-2270dw-printer-using-a-raspberry-pi.md b/posts/printing-on-the-brother-hl-2270dw-printer-using-a-raspberry-pi.md similarity index 100% rename from posts-md/printing-on-the-brother-hl-2270dw-printer-using-a-raspberry-pi.md rename to posts/printing-on-the-brother-hl-2270dw-printer-using-a-raspberry-pi.md diff --git a/posts-md/problem-log-txt.md b/posts/problem-log-txt.md similarity index 100% rename from posts-md/problem-log-txt.md rename to posts/problem-log-txt.md diff --git a/posts-md/qr-backup-2.md b/posts/qr-backup-2.md similarity index 100% rename from posts-md/qr-backup-2.md rename to posts/qr-backup-2.md diff --git a/posts-md/qr-backup-v1-1.md b/posts/qr-backup-v1-1.md similarity index 100% rename from posts-md/qr-backup-v1-1.md rename to posts/qr-backup-v1-1.md diff --git a/posts-md/qr-backup.md b/posts/qr-backup.md similarity index 100% rename from posts-md/qr-backup.md rename to posts/qr-backup.md diff --git a/posts-md/raspberry-pi-comparison.md b/posts/raspberry-pi-comparison.md similarity index 100% rename from posts-md/raspberry-pi-comparison.md rename to posts/raspberry-pi-comparison.md diff --git a/posts-md/relay-music.md b/posts/relay-music.md similarity index 100% rename from posts-md/relay-music.md rename to posts/relay-music.md diff --git a/posts-md/repulsive-dots.md b/posts/repulsive-dots.md similarity index 100% rename from posts-md/repulsive-dots.md rename to posts/repulsive-dots.md diff --git a/posts-md/roasted-chickpeas.md b/posts/roasted-chickpeas.md similarity index 100% rename from posts-md/roasted-chickpeas.md rename to posts/roasted-chickpeas.md diff --git a/posts-md/running-a-forge-server-on-headless-linux.md b/posts/running-a-forge-server-on-headless-linux.md similarity index 100% rename from posts-md/running-a-forge-server-on-headless-linux.md rename to posts/running-a-forge-server-on-headless-linux.md diff --git a/posts-md/scan-organizer.md b/posts/scan-organizer.md similarity index 100% rename from posts-md/scan-organizer.md rename to posts/scan-organizer.md diff --git a/posts-md/scheme-interpreter.md b/posts/scheme-interpreter.md similarity index 100% rename from posts-md/scheme-interpreter.md rename to posts/scheme-interpreter.md diff --git a/posts-md/screen-and-tmux-ides.md b/posts/screen-and-tmux-ides.md similarity index 100% rename from posts-md/screen-and-tmux-ides.md rename to posts/screen-and-tmux-ides.md diff --git a/posts-md/known-good/scroll-props.md b/posts/scroll-props.md similarity index 100% rename from posts-md/known-good/scroll-props.md rename to posts/scroll-props.md diff --git a/posts-md/setting-up-ssl-certificates-using-startssl.md b/posts/setting-up-ssl-certificates-using-startssl.md similarity index 100% rename from posts-md/setting-up-ssl-certificates-using-startssl.md rename to posts/setting-up-ssl-certificates-using-startssl.md diff --git a/posts-md/software-section.md b/posts/software-section.md similarity index 100% rename from posts-md/software-section.md rename to posts/software-section.md diff --git a/posts-md/sql-views.md b/posts/sql-views.md similarity index 100% rename from posts-md/sql-views.md rename to posts/sql-views.md diff --git a/posts-md/steak-tartare-3.md b/posts/steak-tartare-3.md similarity index 100% rename from posts-md/steak-tartare-3.md rename to posts/steak-tartare-3.md diff --git a/posts-md/storage-prices-2019-07.md b/posts/storage-prices-2019-07.md similarity index 100% rename from posts-md/storage-prices-2019-07.md rename to posts/storage-prices-2019-07.md diff --git a/posts-md/storage-prices-2020-01.md b/posts/storage-prices-2020-01.md similarity index 100% rename from posts-md/storage-prices-2020-01.md rename to posts/storage-prices-2020-01.md diff --git a/posts-md/storage-prices-2022-07.md b/posts/storage-prices-2022-07.md similarity index 100% rename from posts-md/storage-prices-2022-07.md rename to posts/storage-prices-2022-07.md diff --git a/posts-md/storage-prices-2023-01.md b/posts/storage-prices-2023-01.md similarity index 100% rename from posts-md/storage-prices-2023-01.md rename to posts/storage-prices-2023-01.md diff --git a/posts-md/streaming-linux-twitch-using-ffmpeg-and-alsa.md b/posts/streaming-linux-twitch-using-ffmpeg-and-alsa.md similarity index 100% rename from posts-md/streaming-linux-twitch-using-ffmpeg-and-alsa.md rename to posts/streaming-linux-twitch-using-ffmpeg-and-alsa.md diff --git a/posts-md/stylish.md b/posts/stylish.md similarity index 100% rename from posts-md/stylish.md rename to posts/stylish.md diff --git a/posts-md/talk-in-debian.md b/posts/talk-in-debian.md similarity index 100% rename from posts-md/talk-in-debian.md rename to posts/talk-in-debian.md diff --git a/posts-md/terminal-goal-rationality-techniques.md b/posts/terminal-goal-rationality-techniques.md similarity index 100% rename from posts-md/terminal-goal-rationality-techniques.md rename to posts/terminal-goal-rationality-techniques.md diff --git a/posts-md/testing-scrapers-faster.md b/posts/testing-scrapers-faster.md similarity index 100% rename from posts-md/testing-scrapers-faster.md rename to posts/testing-scrapers-faster.md diff --git a/posts-md/the-bible-translated-to-the-new-latin.md b/posts/the-bible-translated-to-the-new-latin.md similarity index 100% rename from posts-md/the-bible-translated-to-the-new-latin.md rename to posts/the-bible-translated-to-the-new-latin.md diff --git a/posts-md/known-good/the-double-lives-of-books.md b/posts/the-double-lives-of-books.md similarity index 100% rename from posts-md/known-good/the-double-lives-of-books.md rename to posts/the-double-lives-of-books.md diff --git a/posts-md/the-life-changing-magic-of-tidying-up.md b/posts/the-life-changing-magic-of-tidying-up.md similarity index 100% rename from posts-md/the-life-changing-magic-of-tidying-up.md rename to posts/the-life-changing-magic-of-tidying-up.md diff --git a/posts-md/time-log-transcribed.md b/posts/time-log-transcribed.md similarity index 100% rename from posts-md/time-log-transcribed.md rename to posts/time-log-transcribed.md diff --git a/posts-md/time-management-optimizers-satisficers-minimizers.md b/posts/time-management-optimizers-satisficers-minimizers.md similarity index 100% rename from posts-md/time-management-optimizers-satisficers-minimizers.md rename to posts/time-management-optimizers-satisficers-minimizers.md diff --git a/posts-md/timelog-analysis.md b/posts/timelog-analysis.md similarity index 100% rename from posts-md/timelog-analysis.md rename to posts/timelog-analysis.md diff --git a/posts-md/tiny-cute-vampire-bat.md b/posts/tiny-cute-vampire-bat.md similarity index 100% rename from posts-md/tiny-cute-vampire-bat.md rename to posts/tiny-cute-vampire-bat.md diff --git a/posts-md/tty-audit-logs.md b/posts/tty-audit-logs.md similarity index 100% rename from posts-md/tty-audit-logs.md rename to posts/tty-audit-logs.md diff --git a/posts-md/known-good/understanding-gzip-2.md b/posts/understanding-gzip-2.md similarity index 100% rename from posts-md/known-good/understanding-gzip-2.md rename to posts/understanding-gzip-2.md diff --git a/posts-md/url-handlers-in-linux.md b/posts/url-handlers-in-linux.md similarity index 100% rename from posts-md/url-handlers-in-linux.md rename to posts/url-handlers-in-linux.md diff --git a/posts-md/usb-flash-longevity-testing-year-2.md b/posts/usb-flash-longevity-testing-year-2.md similarity index 100% rename from posts-md/usb-flash-longevity-testing-year-2.md rename to posts/usb-flash-longevity-testing-year-2.md diff --git a/posts-md/what-i-know-about-sleep-schedules.md b/posts/what-i-know-about-sleep-schedules.md similarity index 100% rename from posts-md/what-i-know-about-sleep-schedules.md rename to posts/what-i-know-about-sleep-schedules.md diff --git a/posts-md/when-is-rick-and-morty-out-season-4-episode-6.md b/posts/when-is-rick-and-morty-out-season-4-episode-6.md similarity index 100% rename from posts-md/when-is-rick-and-morty-out-season-4-episode-6.md rename to posts/when-is-rick-and-morty-out-season-4-episode-6.md diff --git a/posts-md/known-good/whiteboard-partition.md b/posts/whiteboard-partition.md similarity index 100% rename from posts-md/known-good/whiteboard-partition.md rename to posts/whiteboard-partition.md diff --git a/posts-md/known-good/whoosh.md b/posts/whoosh.md similarity index 100% rename from posts-md/known-good/whoosh.md rename to posts/whoosh.md diff --git a/posts-md/whsh.md b/posts/whsh.md similarity index 100% rename from posts-md/whsh.md rename to posts/whsh.md diff --git a/posts-md/wip-dead-tree-publishing-2.md b/posts/wip-dead-tree-publishing-2.md similarity index 100% rename from posts-md/wip-dead-tree-publishing-2.md rename to posts/wip-dead-tree-publishing-2.md diff --git a/posts-md/known-good/wip-dead-tree-publishing-3.md b/posts/wip-dead-tree-publishing-3.md similarity index 100% rename from posts-md/known-good/wip-dead-tree-publishing-3.md rename to posts/wip-dead-tree-publishing-3.md diff --git a/posts-md/known-good/wip-dead-tree-publishing-4.md b/posts/wip-dead-tree-publishing-4.md similarity index 100% rename from posts-md/known-good/wip-dead-tree-publishing-4.md rename to posts/wip-dead-tree-publishing-4.md diff --git a/posts-md/known-good/wip-dead-tree-publishing.md b/posts/wip-dead-tree-publishing.md similarity index 100% rename from posts-md/known-good/wip-dead-tree-publishing.md rename to posts/wip-dead-tree-publishing.md diff --git a/posts-md/known-good/xp-boot-usb-stick.md b/posts/xp-boot-usb-stick.md similarity index 100% rename from posts-md/known-good/xp-boot-usb-stick.md rename to posts/xp-boot-usb-stick.md diff --git a/posts-md/year-in-review.md b/posts/year-in-review.md similarity index 100% rename from posts-md/year-in-review.md rename to posts/year-in-review.md diff --git a/posts-md/youtube-autodl.md b/posts/youtube-autodl.md similarity index 100% rename from posts-md/youtube-autodl.md rename to posts/youtube-autodl.md diff --git a/posts-md/known-good/zorchpad-keyboard-update.md b/posts/zorchpad-keyboard-update.md similarity index 100% rename from posts-md/known-good/zorchpad-keyboard-update.md rename to posts/zorchpad-keyboard-update.md diff --git a/posts-md/zorchpad-update-cardboard-mockup-mk1.md b/posts/zorchpad-update-cardboard-mockup-mk1.md similarity index 100% rename from posts-md/zorchpad-update-cardboard-mockup-mk1.md rename to posts/zorchpad-update-cardboard-mockup-mk1.md diff --git a/posts-md/zorchpad-update-first-3d-print.md b/posts/zorchpad-update-first-3d-print.md similarity index 100% rename from posts-md/zorchpad-update-first-3d-print.md rename to posts/zorchpad-update-first-3d-print.md diff --git a/posts-md/zorchpad-update-keyboard.md b/posts/zorchpad-update-keyboard.md similarity index 100% rename from posts-md/zorchpad-update-keyboard.md rename to posts/zorchpad-update-keyboard.md diff --git a/problem_survey.txt b/problem_survey.txt deleted file mode 100644 index f387751..0000000 --- a/problem_survey.txt +++ /dev/null @@ -1,34 +0,0 @@ -Problem Survey - - -++++++++++++++ wp-code-block should be wp-block-code (installing-email-with-postfix-and-dovecot) -++++++ gallery layout information is being dropped -- rows (banh-chung) -image captions are being converted to

text. originals are: - +

(painting) - + div.wp-block-cover - +++++++ p.wp-caption-text (whiteboard-partition) - wrapped inside div.wp-caption, which adds whitespace -++ Original has no

after , just unwrapped text (diy-keyboards-and-how-keyboards-work) -+ image alignright dropped (whiteboard-partition) -+++ yt embed missing a little surrounding space. Original was wrapped in a figure.wp-block-embed.is-type-video.is-provider-youtube.wp-block-embed-youtube.wp-embed-aspect-16-9.wp-has-aspect-ratio (relay-music) -+++++ img.alignnone lost, which was providing a border (a-pixel-art): dontfix -++ img.aligncenter lost, which was providing centering (the-double-lives-of-books) and border (dontfix) -+
after img.alignnone displaying differently (steak-tartare-3) -+ figure class_ is being set, not class (steak-tartare-3) -++ image specified height and width (cron-email-and-sending-email-to-only-one-address) -+++

with an empty link. this is a problem in the markdown->HTML step for image links (moreorcs-com.html) - +++ or empty

only (printable-todo-list) -++ markdown bold breaking across lines. HTML->markdown broken (2020-books) -++++++ markdown version has additional whitespace are

 inside  (mail-filtering-with-dovecot)
-+  converted to  for table (understanding-gzip-2)
-++ figure.wp-block-table should wrap a table (understanding-gzip-2)
-+ several images in a single 
 tag should be converted to a gallery row (default-twitter-icon)
-+ markdown alt text is having markdown applied inside it. this is a bug in the markdown->HTML step (default-twitter-icon)
-+ video got deleted (e-ink-laptop)
-+ 
 being generated inside 
, and adding too much space
-
-+  got lost (archiving-twitter)
-+  messed up (xp-boot-usb-stick) -- problem in markdown to html conversion, i think
-+ make font bigger (hack-a-day-hack-a-hang)
-+ literal * messing with bolding (2022-books)
-+ Link inside quote should be bolded (the-bible-translated-to-the-new-latin)
diff --git a/templates/postcombined.mustache.html b/templates/postcombined.mustache.html
deleted file mode 100644
index 31d252e..0000000
--- a/templates/postcombined.mustache.html
+++ /dev/null
@@ -1,81 +0,0 @@
-
-{{title}}
-
-
-
-
-
-    original blog
-    html source (local)
-    markdown source (local)
-    
-        Images:
-        diff / 
-        html / 
-        md
-    
-
-
-    
-        
-    
-    
-        
-    
-
diff --git a/visualdiff-display.py b/visualdiff-display.py
deleted file mode 100644
index 4b32c6b..0000000
--- a/visualdiff-display.py
+++ /dev/null
@@ -1,49 +0,0 @@
-"""
-Display top problems
-"""
-
-import csv
-import json
-import multiprocessing
-import os
-import pathlib
-import subprocess
-import sys
-import time
-
-tqdm = lambda x, **kw: x
-if sys.stdout.isatty():
-    try:
-        from tqdm import tqdm
-    except ImportError:
-        pass
-
-def unsorted_parallel_map(f, lst, n=10):
-    with multiprocessing.Pool(n) as p:
-        yield from tqdm(p.imap_unordered(f, lst), total=len(lst))
-
-
-def sort_order(x):
-    return [
-        x["pixel-perfect?"] == "False",
-        int(x["height-difference"]),
-        int(x["pixels-different"]),
-        x["post-id"],
-    ]
-
-def open_webpage(url):
-    subprocess.run(["chromium", url])
-
-if __name__ == "__main__":
-    with open("visual-diff.csv", "r") as _csv:
-        csvfile = csv.DictReader(_csv, dialect="excel")
-        #csvfile.writerow(["post-id", "url", "local-url", "pixel-perfect?", "html-screenshot", "markdown-screenshot", "diff-screenshot", "height-difference", "pixels-different"])
-        rows = list(csvfile)
-
-    ordered = sorted(rows, key=sort_order, reverse=True)
-    ordered = [x for x in ordered if x["pixel-perfect?"] == "False"]
-    #for x in ordered[:10]:
-    #    print(x["post-id"], x["height-difference"], x["pixels-different"], x["pixel-perfect?"])
-    for x in ordered:#[:30]:
-        print(x["local-url"])
-        open_webpage(x["local-url"])
diff --git a/visualdiff.py b/visualdiff.py
deleted file mode 100644
index c4c7e34..0000000
--- a/visualdiff.py
+++ /dev/null
@@ -1,79 +0,0 @@
-"""
-Compare markdown and HTML versions of posts.
-Assumes they are already generated
-"""
-
-import csv
-import json
-import multiprocessing
-import os
-import pathlib
-import subprocess
-import sys
-import time
-
-tqdm = lambda x, **kw: x
-if sys.stdout.isatty():
-    try:
-        from tqdm import tqdm
-    except ImportError:
-        pass
-
-def unsorted_parallel_map(f, lst, n=10):
-    with multiprocessing.Pool(n) as p:
-        yield from tqdm(p.imap_unordered(f, lst), total=len(lst))
-
-def pixel_compare(*args):
-    result = subprocess.run(["node", "pixel-compare.js"] + list(args), capture_output=True)
-    identical = result.returncode == 0
-    output = result.stdout.decode('utf8')
-    try:
-        result = json.loads(output)
-    except:
-        print(args, output, file=sys.stderr)
-        raise
-
-    return identical, result
-
-def blog_articles():
-    return sorted(x.stem for x in pathlib.Path("posts-html").iterdir())
-
-def compare(post_id, save=False):
-    url = "https://blog2.za3k.com/posts/{}.html".format(post_id)
-    local_url = "file:///home/zachary/blog/public/posts/{}.html".format(post_id)
-    html_path = "public/posts/{}.orig.html".format(post_id)
-    markdown_path = "public/posts/{}.md.html".format(post_id)
-    html_screenshot_path = "screenshots/{}.html.png".format(post_id)
-    markdown_screenshot_path = "screenshots/{}.md.png".format(post_id)
-    visual_diff_path = "screenshots/{}.diff.png".format(post_id)
-    pixel_identical, ret = pixel_compare(html_path, markdown_path, html_screenshot_path, markdown_screenshot_path, visual_diff_path)
-    return [post_id, url, local_url, pixel_identical, html_screenshot_path, markdown_screenshot_path, visual_diff_path, ret["heightDifference"], ret["pixelsDifferent"]]
-
-if __name__ == "__main__":
-
-    start_time = time.time()
-    with open("visual-diff.csv", "w") as _csv:
-        csvfile = csv.writer(_csv, dialect="excel")
-        csvfile.writerow(["post-id", "url", "local-url", "pixel-perfect?", "html-screenshot", "markdown-screenshot", "diff-screenshot", "height-difference", "pixels-different"])
-        rows = sorted(unsorted_parallel_map(compare, blog_articles()))
-        csvfile.writerows(rows)
-    time_elapsed = time.time() - start_time
-
-    total = len(rows)
-    pixel_identical = len([x for x in rows if x[3]])
-    not_identical = total - pixel_identical
-    example_failure = sorted([x[0] for x in rows if not x[3]])[0]
-
-    print("        Progress Tracker\n")
-    print("            DIFFERENT      SAME    ")
-    print("          |-----------|-----------|")
-    print("          |    {: >3.0f}%   |    {: >3.0f}%   | 100%".format(not_identical/total*100, pixel_identical/total*100))
-    print("          |-----------|-----------|")
-    print("          |    {: >3d}    |    {: >3d}    | {: >3d}".format(not_identical, pixel_identical, total))
-    print("          |-----------|-----------|")
-    print()
-    print("Time: {:.0f}s".format(time_elapsed))
-    print("Time per file: {:.2f}s".format(time_elapsed/total))
-    print("Next failure: {}".format(example_failure))
-
-    os.system("rm -r /tmp/puppeteer_dev_chrome_profile-X*")
diff --git a/wordpress2frontmatter.py b/wordpress2frontmatter.py
deleted file mode 100644
index 54bcd53..0000000
--- a/wordpress2frontmatter.py
+++ /dev/null
@@ -1,67 +0,0 @@
-import bs4
-from datetime import datetime
-import yaml
-
-from pathlib import Path
-
-INPUT_DIR = Path("/home/zachary/blog.za3k.com")
-OUTPUT_DIR = Path("/home/zachary/blog/posts-html")
-IMAGES = OUTPUT_DIR / 'images'
-
-BLACKLIST={"wp-json", "feed"}
-def posts():
-    for possible_post in INPUT_DIR.iterdir():
-        if possible_post.is_dir() and possible_post.name not in BLACKLIST:
-            possible_post /= "index.html"
-            if possible_post.is_file():
-                yield possible_post
-
-def parse_date(s):
-    # 2023-07-17T13:58:49-07:00
-    return datetime.strptime(s, "%Y-%m-%dT%H:%M:%S%z")
-        
-def scrape_post(post):
-    html = bs4.BeautifulSoup(post, 'html.parser')
-    article = html.find('article')
-    post = article.find('div', class_="entry-content")
-
-    for x in html.select('ol.commentlist > li.pingback'):
-        x.extract()
-    comments = html.find('ol', class_="commentlist")
-    if comments and len(comments.find_all('li')) == 0:
-        comments = None
-
-    result = {}
-    result["html_content"] = str(post)
-    result["html_comments"] = (str(comments) if comments else "")
-    result["title"] = article.find('h1', class_="entry-title").get_text()
-    assert result["title"] == html.find('title').get_text()
-    result["date"] = parse_date(html.find('time', class_="published")['datetime'])
-    if updated := html.find('time', class_="updated"):
-        result["updated"] = parse_date(updated['datetime'])
-    result["wordpress_id"] = int(article['id'].removeprefix("post-"))
-    result["categories"] = [link.get_text() for link in article.select(".bl_categ a[rel=tag]")]
-    result["tags"] = [link.get_text() for link in article.select(".bl_posted a[rel=tag]")]
-    result["source"] = "wordpress"
-    result["author"] = article.find(rel="author").get_text()
-    result["markup"] = "html"
-
-    return result
-
-def output_post(post):
-    content = "\n" + post.pop("html_content") + "\n\n\n" + post.pop("html_comments")
-    front_matter = yaml.dump(post)
-    return "---\n{front_matter}---\n{content}".format(content=content, front_matter=front_matter)
-
-import sys
-if __name__ == "__main__":
-    for post_path in posts():
-        slug = post_path.parts[-2] 
-        output_path = OUTPUT_DIR / (slug + ".html")
-        with open(post_path, "r") as f:
-            post = f.read()
-        parse = scrape_post(post)
-        parse["wordpress_slug"] = slug
-        output = output_post(parse)
-        with open(output_path, "w") as f:
-            f.write(output)