I'm often looking for more climbing partners, and people posting on the mountainproject.com forum is a great way to find people to climb with. Here's a little thing I wrote that emails me when someone posts on the forum for the north east.

This script runs on heroku every 20 minutes. It loads the forum posts, and loads a saved version of the forum from an Amazon s3 bucket, a great free online storage tool. If it finds something new, it emails the new posts to me.

import requests, os, bs4, pickle, smtplib
from email.mime.text import MIMEText
import os, os.path
import time
from nonsense import user, password, ID, SECRET

import boto
import boto.s3.connection
from boto.s3.key import Key

pfile = 'partners.p' #the file that stores the cached posts.

def load_partner_forum():
    url = 'https://www.mountainproject.com/forum/105987244/north-eastern-states-partners'
    r = requests.get(url)
    soup = bs4.BeautifulSoup(r.text, "html5lib")
    t = soup.find("table", {'id':'forum-table'})
    t = t.find('tbody')
    rows = t.find_all('tr')
    posts = []
    for row in rows:
        posts.append(format_row(row))
    return posts

def load_previous():
    filename = pfile
    conn = boto.s3.connect_to_region('us-east-1',
    aws_access_key_id = ID,
    aws_secret_access_key = SECRET,
    calling_format = boto.s3.connection.OrdinaryCallingFormat(),
    )

    bucket = conn.get_bucket('mpscrapetest')
    k =  Key(bucket, filename)
    k.get_contents_to_filename(filename)

    with open(filename,'rb') as f:
        try:
            s3data = pickle.load(f)
        except:
            s3data = []
    return s3data

def local_to_s3():
    filename = pfile
    conn = boto.s3.connect_to_region('us-east-1',
    aws_access_key_id = ID,
    aws_secret_access_key = SECRET,
    # host = 's3-website-us-east-1.amazonaws.com',
    # is_secure=True,               # uncomment if you are not using ssl
    calling_format = boto.s3.connection.OrdinaryCallingFormat(),
    )

    bucket = conn.get_bucket('mpscrapetest')
    k = bucket.new_key(filename)
    k.set_contents_from_filename(filename)

def save_new_posts(new_posts):
    with open(pfile, 'wb') as f:
        pickle.dump(posts,f)
    save(pfile)

def format_row(row):
    row = row.find('td')
    title =  row.find('a').get_text()
    link = row.find('a').attrs['href']
    name = row.find_all("span")[1].contents[0]
    name = name.strip().strip('\xe2').strip()
    return [title,name, link]

def make_pretty(new_posts):
    email_text = ""
    for post in new_posts:
        line = ""
        for info in post:
            line += info + ", "
        line += "\n"
        email_text += line
    return str(email_text)

def check_for_new_posts(posts, previous_posts):
    new_posts = []
    if not previous_posts:
        return posts
    else:
        for post in posts:
            if post not in previous_posts:
                new_posts.append(post)
    return new_posts

def email_news(new_posts):
    msg = MIMEText(make_pretty(new_posts))
    FROM = 'markdaily62@gmail.com'
    TO = 'markdaily62@gmail.com'

    msg['Subject'] = 'Partners'
    msg['From'] = FROM
    msg['To'] = TO
    try:
        server = smtplib.SMTP('smtp.gmail.com:587')
        server.ehlo()
        server.starttls()
        server.login(user, password)
        server.sendmail(FROM,TO,msg.as_string())
        server.close()
        print('Email successfully sent!')
    except Exception as e:
        print ('Problem emailing.')
        print (e)

def update_localfile(posts):
    with open(pfile, 'wb') as f:
        pickle.dump(posts, f)

def check_partner_forum():
    posts = load_partner_forum()
    previous_posts = load_previous()
    unseen = check_for_new_posts(posts, previous_posts)
    if unseen:
        update_localfile(posts)
        local_to_s3()
        email_news(unseen)
    else:
        print('Nothing new!')

if __name__ == "__main__":
    check_partner_forum()

Nothing new!