E-mail archiving in DT is great and works very well. One thing I was missing though was the ability to automatically import my new mail. I have a simple archiving strategy: delete nothing, archive everything (this might not be everyone’s preference!)
I have a simple rule in Fastmail to add a “Year/2022” label to every incoming mail (meaning that basically everything I’ve received is labeled). With a Python script I simply check hourly for new mail in the 2022 mailbox + in my Sent folder. It also makes use of the "Extract attachment" script so all my incoming attachments are automatically searchable in DT.
For me this approach goes a long way in making DT my ‘single source of thruth’ for any document I have received in my e-mail, downloaded, clipped etc. I just search something in my DT databases and can almost immediately find it and e.g. send it to someone else or use it in a document I’m building.
Notes / bugs:
I’ve run into some weird situations looking like a race condition where DT tries to execute the smart rule (or tries to execute it too soon) while the database is not (fully) opened. Which is why I’ve included a simple database check in the Applescript below. Normally smart rules should only run when the database is fully opened.
We should be able to call a smart rule from within a smart rule, but that behavior is currently not working in 3.8.3 which is why I’m using a bit of a workaround: including the script and calling performsmartrule(theRecords) directly. This should be possible in 3.8.4 again. As a workaround the import tags are used to determine which records to process for replacing attachments.
Applescript: Auto import new mail
use AppleScript version "2.4" -- Yosemite (10.10) or later
use scripting additions
property pythonCmd : "/Users/mdbraber/.pyenv/shims/python3"
property pythonScriptPath : "/Users/mdbraber/iCloud Drive/Coding/Devonthink import mail/"
property pythonScriptName : "import-mail-to-dt.py"
property importCmd : pythonCmd & " " & quoted form of (pythonScriptPath & pythonScriptName)
--
property replaceScript : load script POSIX file "/Users/mdbraber/Library/Application Scripts/com.devon-technologies.think3/Smart Rules/Replace mail attachments.scpt"
on run
tell application id "DNtp"
my performSmartRule(selection as list)
end tell
end run
on performSmartRule(theRecords)
tell application id "DNtp"
try
-- Check if the database is open, because sometimes when starting
-- DT it isn't open yet while the smart rule already runs (which is
-- weird but might have to do with the opening taking longer?)
set theDatabase to database "Mail"
set theOutput to do shell script importCmd
-- This is a workaround because 'perform smart rule' doesn't work
-- from smart rules in DT 3.8.3 so we're including our script and
-- using tagged records. Ugly, but it works for now. This should
-- be fixed in DT 3.8.4
set theRecords to search "tags:import scope:\"Mail\""
-- This script replaces attachments and also strips the import tag
-- (for now)
replaceScript's performSmartRule(theRecords)
on error error_message number error_number
if error_number is not -128 then log message error_message
end try
end tell
end performSmartRule
Python: import-mail-to-dt.py
#!/usr/bin/env python3
import imaplib
import email
from email.header import decode_header
from email.header import make_header
import uuid
import subprocess
import sys
import os
import logging
import keyring
import re
import datetime
import traceback
# login username
username = "your@email.com"
# login password (using keyring to save credentials)
# set password by issueing the command `keyring.set_password("import-mail-to-dt", username, "s3cr3tpa$sw0rd"
password = keyring.get_password("import-mail-to-dt", username)
imap_server = "imap.myprovider.com"
imap_port = 993
# get current year (my mailboxes are year-based)
year = datetime.datetime.now().date().strftime("%Y")
# DT database name
database = "Mail"
# group in DT database
group = "/" + year
# current path of this script
script_path = os.path.dirname(__file__)
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(message)s')
try:
mail = imaplib.IMAP4_SSL(imap_server,imap_port)
mail.login(username,password)
count = 0
for mailbox in [year, "Sent"]:
# I use year-based archive mailboxes
if mailbox == year:
mailbox_name = "Year/" + year
else:
mailbox_name = mailbox
uid_filename = os.path.join(script_path, ".uid." + mailbox)
uid_file = open(uid_filename, "r")
uid_line = uid_file.readline().split(" ")
uidnext_prev = int(uid_line[0])
uidvalidity_prev = int(uid_line[1])
status = mail.status(mailbox_name, '(UIDNEXT UIDVALIDITY)')
if status[0] == "OK":
uidnext_now = int(re.search(r"UIDNEXT (\d*)",str(status[1][0].decode())).group(1))
uidvalidity_now = int(re.search(r"UIDVALIDITY (\d*)",str(status[1][0].decode())).group(1))
if not (uidvalidity_prev == uidvalidity_now):
exit("UIDVALIDITY of " + mailbox_name + " has changed - stopping")
else:
exit("Can't get mailbox status of " + mailbox_name)
if uidnext_now > uidnext_prev:
mail.select(mailbox_name,True)
type, data = mail.search(None, 'UID ' + str(uidnext_prev) + ':*')
id_list = data[0].split()
for num in id_list:
resp, data = mail.fetch(num, '(RFC822)')
raw_email = data[0][1]
raw_email_string = raw_email.decode('utf-8')
msg = email.message_from_string(raw_email_string)
filename = os.path.join(script_path, str(uuid.uuid4()) + ".eml")
file = open(filename, "w")
a = file.write(str(msg))
file.close()
if os.path.exists(filename):
script = '''
tell application id "DNtp"
if it is running
set theGroup to create location "{group}" in database "{database}"
set theRecord to import POSIX path of "{filename}" to theGroup
set theMetadata to meta data of theRecord
set theSubject to |kMDItemSubject| of theMetadata
set name of theRecord to theSubject
set unread of theRecord to false
set modification date of theRecord to creation date of theRecord
-- This does not work in DT 3.8.3 so we use a workaround by
-- adding an 'import tag' (see calling AppleScript for details)
--perform smart rule "Mail: replace attachments" record theRecord
set tags of theRecord to {{"import"}}
else
error "DEVONthink not running" number -10000
end if
end tell
'''
proc = subprocess.Popen(['osascript', '-'], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
proc_input = bytes(script.format(group=group,database=database,filename=filename),'utf-8')
proc.communicate(input=proc_input)
if not proc.returncode == 0:
exit("Something went wrong importing the e-mail - is DT running?")
subject = str(make_header(decode_header(msg['subject']))).replace("\\n", "")
logging.info("Downloading %s",subject)
count = count + 1
os.remove(filename)
uid_file = open(uid_filename, "w")
a = uid_file.write(str(uidnext_now) + " " + str(uidvalidity_now))
uid_file.close()
mail.close()
if count > 0:
logging.info("Downloaded %s messages",count)
mail.logout()
except Exception as e:
traceback.print_exc()
print (str(e))
