Blog

poetry init
poetry add motor pydantic
poetry add mkdocs-material mkdocstrings[python] mypy ruff poethepoet pytest pytest-asyncio pytest-random-order --group dev

[project]
name = "motorhead"
description = "Async MongoDB with vanilla Pydantic v2+ - made easy."
readme = "README.md"
license = { text = "MIT" }
authors = [
    { name = "Peter Volf", email = "[email protected]" },
]
requires-python = ">=3.10"
dependencies = ["pydantic", "motor"]
classifiers = [
    "Intended Audience :: Information Technology",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3",
    "Development Status :: 4 - Beta",
    "Topic :: Internet",
    "Topic :: Software Development :: Libraries",
    "Topic :: Software Development",
    "Typing :: Typed",
    "Environment :: Web Environment",
    "Framework :: FastAPI",
    "Intended Audience :: Developers",
    "License :: OSI Approved :: MIT License",
    "Topic :: Internet :: WWW/HTTP",
]

[project.urls]
homepage = "https://github.com/volfpeter/motorhead"
documentation = "https://volfpeter.github.io/motorhead"
tracker = "https://github.com/volfpeter/motorhead/issues"

[tool.poetry]
name = "motorhead"
version = "0.2403.0"
description = "Async MongoDB with vanilla Pydantic v2+ - made easy."
authors = ["Peter Volf <[email protected]>"]
readme = "README.md"
packages = [{include = "motorhead"}]

[tool.poetry.dependencies]
python = "^3.10"
motor = "^3.1.0"
pydantic = "^2.1.0"

[tool.poetry.group.dev.dependencies]
mkdocs-material = "^9.5.19"
mkdocstrings = {extras = ["python"], version = "^0.25.0"}
mypy = "^1.10.0"
ruff = "^0.4.2"
poethepoet = "^0.26.0"
pytest = "^8.2.0"
pytest-asyncio = "^0.23.6"
pytest-docker = "^3.1.1"
pytest-random-order = "^1.1.1"

[tool.mypy]
strict = true
show_error_codes = true
exclude = ["tree_app"]

[[tool.mypy.overrides]]
module = ["motor.*"]
ignore_missing_imports = true

[tool.ruff]
line-length = 108
lint.exclude = [
    ".git",
    ".mypy_cache",
    ".pytest_cache",
    ".ruff_cache",
    ".venv",
    "dist",
    "docs",
]
lint.select = [
    "E",  # pycodestyle errors
    "W",  # pycodestyle warnings
    "F",  # pyflakes
    "I",  # isort
    "S",  # flake8-bandit - we must ignore these rules in tests
    "C",  # flake8-comprehensions
    "B",  # flake8-bugbear
]

[tool.ruff.lint.per-file-ignores]
"tests/**/*" = ["S101"]  # S101: use of assert detected

[tool.pytest.ini_options]
addopts = "--random-order"

[tool.poe.tasks]
serve-docs = "mkdocs serve"
check-format = "ruff format --check ."
lint = "ruff check ."
mypy = "mypy ."
format = "ruff format ."
lint-fix = "ruff . --fix"
test = "python -m pytest tests --random-order"

static-checks.sequence = ["lint", "check-format", "mypy"]
static-checks.ignore_fail = "return_non_zero"

[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

test -d /backup || (mkdir /backup; chmod 700 /backup)

cat << 'EOF2' > /backup/borg.sh
#!/usr/bin/env bash

# BorgBackup Script, v1.0, 2024-04-09, by @stoege

# Remote server details
REMOTE_USER="borguser"
REMOTE_HOST="your.remote.borg.server"
REMOTE_REPO="mysamplerepo"

# Local directory to backup
LOCAL_DIR="/"

# List of directories to exclude
EXCLUDE_DIRS=(
    "*/.cache/"
    "/tmp"
    "/restore"
)

# Set PassPhrase for the Backup Encryption (run: pwgen 32 1)
export BORG_PASSPHRASE="your-super-strong-password"
export BORG_RSH='ssh -i /backup/id_ed25519'


# Function to perform full backup
perform_full_backup() {

    # Construct exclude options
    exclude_opts=""
    for dir in "${EXCLUDE_DIRS[@]}"; do
        exclude_opts+="--exclude $dir "
    done

    # Run BorgBackup command
    borg create \
        --verbose \
        --stats \
        --progress \
        --compression lz4 \
        $REMOTE_USER@$REMOTE_HOST:$REMOTE_REPO::'{hostname}-{now:%Y-%m-%d_%H:%M:%S}' \
        $LOCAL_DIR \
        $exclude_opts \
    || borg init -e repokey-blake2 $REMOTE_USER@$REMOTE_HOST:$REMOTE_REPO
}

# Function to perform restore
perform_restore() {

    # Create Restore
    test -d /restore || mkdir /restore

    # Change Dir
    cd /restore
 
    # Run BorgBackup command to restore specific directory
    borg extract \
        --verbose \
        --progress \
        $REMOTE_USER@$REMOTE_HOST:$REMOTE_REPO::$1 \
        $2
}

# Function to list all backups
list_backups() {

    # Run BorgBackup command to list all archives
    borg list $REMOTE_USER@$REMOTE_HOST:$REMOTE_REPO

}


# Function to rotate backups
rotate_backups() {
    # Run BorgBackup command to prune archives based on retention policy
    borg prune \
        --verbose \
        --list \
        --glob-archives "${hostname}*" \
        --keep-hourly=2 \
        --keep-daily=2 \
        --keep-weekly=2 \
        --keep-monthly=2 \
        $REMOTE_USER@$REMOTE_HOST:$REMOTE_REPO
}

# Install on Host
install_myself() {

  # Folder
  f="/backup"

  # Create Directory, copy File
  test -d ${f} || (mkdir ${f}; chmod 700 ${f})
  cp $0 ${f}/
  
  # Inform User
  cat << EOF

# to install in Crontab:
crontab -e
5 6,12,18 * * * cd ${f}; $0 backup >> /var/log/borgbackup.log 2>&1

EOF

}

# Help
show_help() {
  echo "$0 [ backup | list | rotate | install | restore BACKUPNAME /etc ]"
  exit 1
}


# Main script
echo "Starting BorgBackup..."

# Check if BorgBackup is installed
if ! command -v borg &> /dev/null; then
    echo "Error: BorgBackup is not installed. Please install BorgBackup."
    exit 1
fi

# Check if parameter is provided
if [ $# -eq 0 ]; then
  show_help
fi

# Perform action based on parameter
case "$1" in
    "backup")
        echo "Performing full backup..."
        perform_full_backup
        rotate_backups
        ;;
    "restore")
        if [ $# -lt 3 ]; then
            echo "Error: Please specify a Backup Set and directory to restore."
            echo "$0 BACKUP_SET /FOLDER/TO/RESTORE"
            exit 1
        fi
        echo "Performing restore for directory '$3' on set '$2'"
        perform_restore $2 $3
        ;;
    "list")
        echo "Listing all backups..."
        list_backups
        ;;
    "rotate")
        echo "Rotating backups..."
        rotate_backups
        ;;
    "install")
        echo "Install Scripts"
        install_myself
        ;;
    *)
        show_help
        ;;
esac

# Check backup status
if [ $? -eq 0 ]; then
    echo "Action completed successfully."
else
    echo "Action failed."
fi

# Finally done
exit 0

EOF2
chmod 700 /backup/borg.sh

/backup/borg.sh backup

/backup/borg.sh list

/backup/borg.sh restore hostname-date /etc"

poetry init
poetry add twisted
poetry add twisted-mtr

cat << 'EOF' > main.py
#!/usr/bin/env python3

'''
    An example usage for twisted-mtr which initiates multiple async traceroutes
    to multiple IPv4 and IPv6 target IP addresses at the same time. You will
    need to set your source IP addresses correctly and have a working dual
    IPv4/IPv6 networking stack to run this example.

'''

import sys
import signal
import logging
import ipaddress
from twisted.internet import reactor
from twisted_mtr import logger, errors, mtr, utils


log = logger.get_logger('trace', level=logging.DEBUG)


if __name__ == '__main__':

    log.info(f'Starting up...')

    # Find mtr-packet in your path
    mtr_binary_name = 'mtr-packet'
    mtr_binary_path = utils.find_binary(mtr_binary_name)

    # Replace with your local IPv4 address
    # Note that if you set this to an IP address not available on your system
    # your traceroutes will simply all time out
    local_ipv4 = ipaddress.IPv4Address('10.1.2.3')

    # Replace with your local IPv6 address
    # Note that if you set this to an IP address not available on your system
    # your traceroutes will simply all time out
    local_ipv6 = ipaddress.IPv6Address('2404:1:2:3:4:5:6:7')

    # Create the TraceRoute Twisted process object
    app_mtr = mtr.TraceRoute(
        mtr_binary_path=mtr_binary_path,
        local_ipv4=local_ipv4,
        local_ipv6=local_ipv6
    )

    # Bind to the Twisted tractor with the mtr-packet binary
    reactor.spawnProcess(app_mtr, mtr_binary_name, [mtr_binary_path], {})

    # Sets to track the traceroutes that have been dispatched and completed
    requested = set()
    completed = set()

    # Success callback
    def _test_traceroute_callback(timestamp, target_ip, protocol, port, hops):
        log.info(f'Completed traceroute started at {timestamp} to: '
                 f'{target_ip} ({protocol}:{port})')
        completed.add(str(target_ip))
        for (hop_num, hop_ip, microseconds) in hops:
            log.info(f' - {hop_num} {hop_ip} {microseconds}')
        if requested == completed:
            log.info('All traces complete, stopping reactor')
            reactor.stop()

    # Error callback
    def _test_trace_error(counter, joined_request, error, extra):
        log.error(f'Error running traceroute: {error}')
        reactor.stop()

    # Queue up our traceroutes
    target_ip = utils.parse_ip('8.1.1.1')  # No route after a few hops to test
    requested.add(str(target_ip))
    app_mtr.trace(_test_traceroute_callback, _test_trace_error, target_ip)

    target_ip = utils.parse_ip('8.8.8.8')
    requested.add(str(target_ip))
    app_mtr.trace(_test_traceroute_callback, _test_trace_error, target_ip, protocol='udp', port=53)

    target_ip = utils.parse_ip('1.1.1.1')
    requested.add(str(target_ip))
    app_mtr.trace(_test_traceroute_callback, _test_trace_error, target_ip, protocol='tcp', port=53, ttl=3)

    target_ip = utils.parse_ip('2404:6800:4015:802::200e')
    requested.add(str(target_ip))
    app_mtr.trace(_test_traceroute_callback, _test_trace_error, target_ip)

    target_ip = utils.parse_ip('2606:4700::6810:7b60')
    requested.add(str(target_ip))
    app_mtr.trace(_test_traceroute_callback, _test_trace_error, target_ip)

    # Polite hook for control+c to abort the traceroutes before they complete
    def signal_handler(sig, frame):
        sys.stdout.write('\n')  # put the ^C on its own line
        log.info(f'Caught keyboard interrupt, shutting down...')
        reactor.stop()

    signal.signal(signal.SIGINT, signal_handler)

    # Start the Twisted reactor event loop
    log.info(f'Starting event loop...')
    reactor.run()

    # If we reach here the reactor has been stopped, all done
    log.info(f'Goodbye')
EOF

cat << 'EOF' > main.py
from twisted.web import server, resource
from twisted.internet import reactor, endpoints

class Counter(resource.Resource):
    isLeaf = True
    numberRequests = 0

    def render_GET(self, request):

        client_ip = request.getClientAddress().host

        r=request.uri.decode('utf-8')
        if not r =="/favicon.ico":
          self.numberRequests += 1

        request.setHeader(b"content-type", b"text/plain")
        content = u"I am request #{} from {}\n".format(self.numberRequests, client_ip)
        return content.encode("ascii")

endpoints.serverFromString(reactor, "tcp:8080").listen(server.Site(Counter()))
reactor.run()
EOF

poetry init
poetry add twisted
poetry run python main.py

# get Version
brew info xz

# Cleanup Cache
brew cleanup -v -s --prune=all

# Downgrade
brew reinstall xz

# Update
brew update

# Upgrade
brew upgrade

# reboot
reboot

# confirm, 5.4.6 should be fine
xz -V

cat << 'EOF' > /etc/hostname.bridge0
add vether0
add em0
add em1
add em2
up
EOF

echo "up" >> /etc/hostname.em0
echo "up" >> /etc/hostname.em1
echo "up" >> /etc/hostname.em2

cat << 'EOF' > /etc/hostname.vether0
inet  autoconf
inet6 autoconf -temporary -soii
up
EOF

sh /etc/netstart

doas su -
cd /root
ftp https://blog.stoege.net/scripts/{.helpers,upgrade_to_75.sh}
chmod u+x upgrade_to_75.sh

./upgrade_to_75.sh

fsctl loglevel 7

# Backup
cp /etc/pf.conf /etc/pf.conf.bak-$(date "+%s")

# Paste new Ruleset
cat << 'EOF' > /etc/pf.conf

#######################################################################################################################
#
# OpenBSD PF Template
#
#######################################################################################################################

########################
### DEFAULT SETTINGS ###
########################

set block-policy drop
set limit states 100000
set limit table-entries 1000000
set optimization normal
set ruleset-optimization none
set skip on { lo0 enc0  }
set syncookies adaptive (start 25%, end 12%)


########################
### MACROS           ###
########################

# log block
lb  = "log"

# log pass
lp  = "log"


########################
### NORMALIZATION    ###
########################

match inet  scrub (no-df max-mss 1380)
match inet6 scrub (max-mss 1360)


########################
### Block all / uRPF ###
########################

block     log
block in  log quick from urpf-failed label uRPF


############################
### DHCP & IPv6 Stuff    ###
############################

# Allow DHCP
pass      $lp quick inet  proto udp       from  any     port 68   to 255.255.255.255  port 67
pass      $lp quick inet  proto udp       from  any     port 68   to (self)           port 67
pass      $lp quick inet  proto udp       from  (self)  port 67   to any              port 68

# In
pass in   $lp quick inet6 proto ipv6-icmp all                           icmp6-type { unreach toobig neighbrsol neighbradv } keep state

# Out
pass out  $lp quick inet6 proto ipv6-icmp from (self)     to fe80::/10  icmp6-type { echoreq echorep neighbradv neighbrsol routeradv routersol } keep state
pass out  $lp quick inet6 proto ipv6-icmp from (self)     to ff02::/16  icmp6-type { echoreq echorep neighbradv neighbrsol routeradv routersol } keep state

# In
pass in   $lp quick inet6 proto ipv6-icmp from fe80::/10  to fe80::/10  icmp6-type { echoreq neighbradv neighbrsol routeradv routersol } keep state
pass in   $lp quick inet6 proto ipv6-icmp from fe80::/10  to ff02::/16  icmp6-type { echoreq neighbradv neighbrsol routeradv routersol } keep state
pass in   $lp quick inet6 proto ipv6-icmp from ff02::/16  to fe80::/10  icmp6-type { echoreq neighbradv neighbrsol routeradv routersol } keep state
pass in   $lp quick inet6 proto ipv6-icmp from ::         to ff02::/16  icmp6-type { echoreq neighbradv neighbrsol routeradv routersol } keep state


############################
### MyTrust for Hosts    ###
############################

# Allow RFC1918
pass in   $lp quick inet  proto tcp   from { 10/8 172.16/12 192.168/16 }             to (self) port 22

# Allow all Out
pass out  $lp quick


#######################################################################################################################
# End
#######################################################################################################################
EOF

# check & reload
pfctl -nf /etc/pf.confad && pfctl -f /etc/pf.conf

cat << 'EOF' > /etc/hostname.em0
# Public Internet
inet 	autoconf
inet6	autoconf
EOF