diff --git a/fetchmail-FAQ-list-gmail-options-including-oauthbearer-and-app.patch b/fetchmail-FAQ-list-gmail-options-including-oauthbearer-and-app.patch new file mode 100644 index 0000000..e0995be --- /dev/null +++ b/fetchmail-FAQ-list-gmail-options-including-oauthbearer-and-app.patch @@ -0,0 +1,48 @@ +From: Matthew Ogilvie +Date: Sat, 3 Jun 2017 17:57:22 -0600 +Subject: FAQ: list gmail options including oauthbearer and app password +Git-repo: https://gitlab.com/fetchmail/fetchmail.git +Git-commit: dbeee6a0c0fc5392953f38d6f0dcffdeeb8ae141 + +--- + fetchmail-FAQ.html | 24 +++++++++++++++++++++--- + 1 file changed, 21 insertions(+), 3 deletions(-) + +diff --git a/fetchmail-FAQ.html b/fetchmail-FAQ.html +index 82dd92df..027dc9da 100644 +--- a/fetchmail-FAQ.html ++++ b/fetchmail-FAQ.html +@@ -2004,9 +2004,27 @@ sites.)

+

Google has started pushing towards more complex authentication + schemes based on OAuth 2.0 that require clients and users + to jump through quite a few hoops, and use web browsers for signing in. +-If this hinders access to your account through fetchmail, you may need to turn on access for "less secure apps" at https://myaccount.google.com/lesssecureapps.
+-It is disputable whether an application that does not include web ++If this hinders access to your account through fetchmail, you have some ++options:

++ ++

It is disputable whether an application that does not include web + browsing capabilities or heavy-weight libraries is "less secure" as + Google claims.

+ +-- +2.31.1 + diff --git a/fetchmail-add-contrib-fetchnmail-oauth2.py-token-acquisition-u.patch b/fetchmail-add-contrib-fetchnmail-oauth2.py-token-acquisition-u.patch new file mode 100644 index 0000000..245a49b --- /dev/null +++ b/fetchmail-add-contrib-fetchnmail-oauth2.py-token-acquisition-u.patch @@ -0,0 +1,610 @@ +From: Matthew Ogilvie +Date: Thu, 1 Jun 2017 00:09:02 -0600 +Subject: add contrib/fetchnmail-oauth2.py token acquisition utility +Git-repo: https://gitlab.com/fetchmail/fetchmail.git +Git-commit: c82625858682eb2396b6a49da79e403c6f2b018b + +--- + contrib/README | 6 + contrib/fetchmail-oauth2.py | 567 ++++++++++++++++++++++++++++++++++++++++++++ + fetchmail.man | 3 + 3 files changed, 575 insertions(+), 1 deletion(-) + create mode 100755 contrib/fetchmail-oauth2.py + +--- a/contrib/README ++++ b/contrib/README +@@ -181,6 +181,12 @@ sendmail 8.11.0 with multidrop. + + Watchdog script to check whether fetchmail is working in daemon mode. + ++### fetchmail-oauth2.py ++ ++Script to obtain oauth2 access tokens that "fetchmail --auth oauthbearer" ++expects in place of the password. See --help and comments in the ++script, as well as fetchmail --auth documentation. ++ + ### mold-remover.py + + A short python script to remove old read mail from a pop3 mailserver. +--- /dev/null ++++ b/contrib/fetchmail-oauth2.py +@@ -0,0 +1,567 @@ ++#!/usr/bin/python ++# ++# Updates: Copyright 2017 Matthew Ogilvie (mogilvie+fml at zoho.com) ++# - Started with https://github.com/google/gmail-oauth2-tools.git ++# commit 45c39795044c604ed126205806191a8473c0f671 dated ++# 2015-06-09. ++# - Add file interaction (--refresh, --auto_refresh, ++# --obtain_refresh_token_file and related options). ++# - Support both python 2 and 3. ++# - Keeping the same license (below). ++# ++# Copyright 2012 Google Inc. ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++ # http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++ ++############### ++# POSSIBLE IMPROVEMENTS: ++# ++# FUTURE: Explicitly track expiration time of access tokens, ++# and base --auto_refresh on actual expiration time instead of ++# simple age. ++# FUTURE: Add a mode that can print the access token by itself to ++# stdout, presumably piped into fetchmail or similar (either both ++# launched by a wrapper script, this launches fetchmail, or fetchmail ++# launches this). ++# FUTURE: Mix old and new interfaces (or get rid of old interface): ++# Support using a config file to supply some of the details ++# for the original google modes of operation (--generate_oauth2_token, ++# --generate_oauth2_string, --refresh_token, and --test_*). ++# Also support providing sensative data on the command line instead ++# of files for the new modes of operation, despite the lack ++# of security (process listings, .bash_history files, etc). ++# FUTURE: Revise model for how to set permissions on updated files? ++# Preserve existing? Somehow allow setting UID/GID? Warn if files ++# are accessible by anyone but the current user? ++ ++"""Performs client tasks for testing IMAP OAuth2 authentication. ++ ++This documentation and examples is for gmail. For other providers, ++you will likely need to track down appropriate non-default settings ++for auth_url, token_url, and scope. ++ ++To use this script, you'll need to have registered with Google as an OAuth ++application and obtained an OAuth client ID and client secret. ++See https://developers.google.com/identity/protocols/OAuth2 and ++https://developers.google.com/identity/sign-in/web/devconsole-project ++for instructions on registering and for documentation of the APIs ++invoked by this code. ++ ++This script has 2 main modes of operation. ++ ++1. The first mode is used to generate and authorize an OAuth2 token, the ++first step in logging in via OAuth2. ++ ++First, after registering your "application" (above) you should setup a ++configuration file. Use a text editor to do the command-line equivalent of: ++ ++ sed 's/^ *//' > /path/to/oauth2Config.properties << EOF ++ client_id=1038[...].apps.googleusercontent.com ++ client_secret=VWFn8LIKAMC-MsjBMhJeOplZ ++ refresh_token_file=/home/path/to/refresh_token_file ++ access_token_file=/home/path/to/access_token_file ++EOF ++ ++ chmod 600 /path/to/oauth2Config.properties ++ ++Then run the following, and repeat any time the refresh token stops ++working, such as when you change your password. This is interactive ++and requires a web browser to complete: ++ ++ oauth2 -c /path/to/oauth2Config.properties --obtain_refresh_token_file ++ ++The script will converse with Google and generate an oauth request ++token, then present you with a URL you should visit in your browser to ++authorize the token. Once you get the verification code from the Google ++website, enter it into the script, which will then save access and referesh ++tokens to the corresponding files for later use. ++ ++Also, you'll usually need to configure fetchmail by ++including a section like the following in your .fetchmailrc: ++ ++ poll imap.gmail.com protocol imap ++ auth oauthbearer username "USER@gmail.com" ++ passwordfile "/home/path/to/access_token_file" ++ is LOCALUSER here sslmode wrapped sslcertck ++ ++Alternative for debugging: You can also use the original google ++script interface to obtain these tokens without involving files: ++ ++ oauth2 \ ++ --client_id=1038[...].apps.googleusercontent.com \ ++ --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \ ++ --generate_oauth2_token ++ ++ ++----- ++2. The script will generate new access tokens using a refresh token. ++ ++This uses the same config file setup above. ++ ++ oauth2 -c /path/to/oauth2Config.properties --auto_refresh ++ # Or force refresh by using --refresh instead of --auto_refresh. ++ ++ fetchmail -s # or other tools configured to use the access_token_file ++ # And/or call something to update outgoing MTA relay configuration, ++ # if necessary. ++ ++You may put this sequence in a short shell script, ++and configure cron to call it a few times per hour. ++ ++Alternative for debugging: You can also use the original google ++script interface to refresh the token without involving files: ++ ++ oauth2 \ ++ --client_id=1038[...].apps.googleusercontent.com \ ++ --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \ ++ --refresh_token=1/Yzm6MRy4q1xi7Dx2DuWXNgT6s37OrP_DW_IoyTum4YA ++ ++----- ++Google's non-file script interface also supports a few other ++testing modes; see --help. ++""" ++ ++from __future__ import print_function ++import base64 ++import imaplib ++import json ++from optparse import OptionParser ++import smtplib ++import sys ++import os ++import time ++ ++try: ++ import urllib.request as urlopen ++ import urllib.parse as urlparse ++except ImportError: ++ import urllib as urlopen ++ import urllib as urlparse ++ ++try: input = raw_input ++except NameError: pass ++ ++ ++def SetupOptionParser(): ++ # Usage message is the module's docstring. ++ parser = OptionParser(usage=__doc__) ++ parser.add_option('-c', '--config_file', ++ default=None, ++ help='Configuration file for --refresh ' ++ 'and --obtain_refresh_token_file.\n' ++ 'The file should contain 4 (or more) settings, ' ++ 'one per line, or they can also be overridden ' ++ 'by the equivalent options:\n' ++ ' client_id=...\n' ++ ' client_secret=...\n' ++ ' refresh_token_file=/path/to/...\n' ++ ' access_token_file=/path/to/...\n' ++ ' Also max_age_sec, scope, umask, auth_url, and' ++ ' token_url have reasonable defaults for google.') ++ parser.add_option('--auto_refresh', ++ action='store_const', ++ default=None, ++ const=1, ++ dest='refresh', ++ help='Automatically refresh access_token_file, ' ++ 'if older than max_age_sec from ' ++ 'required -c /file/ info.'); ++ parser.add_option('--refresh', ++ action='store_const', ++ const=2, ++ dest='refresh', ++ help='Refresh access_token_file ' ++ 'unconditionally. Requires -c /file/ info.'); ++ parser.add_option('--obtain_refresh_token_file', ++ action='store_true', ++ dest='obtain_refresh_token_file', ++ default=None, ++ help='Update refresh token in file. This is ' ++ 'interactive, and requires ' ++ 'a web browser. Also requires -c /file/ info. ' ++ 'This also saves an initial access_token_file.'); ++ parser.add_option('--client_id', ++ default=None, ++ help='Client ID of the application that is authenticating. ' ++ 'See OAuth2 documentation for details.') ++ parser.add_option('--client_secret', ++ default=None, ++ help='Client secret of the application that is ' ++ 'authenticating. See OAuth2 documentation for ' ++ 'details.') ++ parser.add_option('--access_token_file', ++ default=None, ++ help='File name containing OAuth2 access token') ++ parser.add_option('--refresh_token_file', ++ default=None, ++ help='File name containing OAuth2 refresh token') ++ parser.add_option('--max_age_sec', ++ default=None, # manual default 3000 ++ help='default max file age for --auto_refresh. ' ++ 'Defaults to 3000 (10 minutes short of ' ++ 'normal 3600 sec token expiration).') ++ parser.add_option('--umask', ++ default=None, # manual default 0077 ++ help='default umask for --auto_refresh and ' ++ '--obtain_refresh_token_file. Defaults to 0077.') ++ parser.add_option('--scope', ++ default=None, # manual default='https://mail.google.com/' ++ help='scope for the access token. Multiple scopes can be ' ++ 'listed separated by spaces with the whole argument ' ++ 'quoted. Defaults to https://mail.google.com/') ++ parser.add_option('--auth_url', ++ default=None, # manual default... ++ help='Permission URL for --obtain_refresh_token_file. ' ++ 'Defaults to https://accounts.google.com/o/oauth2/auth.') ++ parser.add_option('--token_url', ++ default=None, # manual default... ++ help='Token URL for --obtain_refresh_token_file,' ++ ' and --refresh. ' ++ 'Defaults to https://accounts.google.com/o/oauth2/token.') ++ parser.add_option('--generate_oauth2_token', ++ action='store_true', ++ dest='generate_oauth2_token', ++ help='(OLD/testing) generates an OAuth2 token for testing.' ++ ' Ignores all files.') ++ parser.add_option('--refresh_token', ++ default=None, ++ help='(OLD/testing) Generate a new access token using' ++ ' this OAuth2 refresh token. Ignores all files.') ++ parser.add_option('--user', ++ default=None, ++ help='(OLD/testing) email address of user whose account' ++ ' is being accessed') ++ parser.add_option('--access_token', ++ default=None, ++ help='(OLD/testing) OAuth2 access token.') ++ parser.add_option('--generate_oauth2_string', ++ action='store_true', ++ dest='generate_oauth2_string', ++ help='(OLD/testing) generates an initial client response' ++ ' string for OAuth2. Ignores all files.') ++ parser.add_option('--test_imap_authentication', ++ action='store_true', ++ dest='test_imap_authentication', ++ help='(OLD/testing) attempts to authenticate to IMAP. ' ++ 'Ignores all files.') ++ parser.add_option('--test_smtp_authentication', ++ action='store_true', ++ dest='test_smtp_authentication', ++ help='(OLD/testing) attempts to authenticate to SMTP. ' ++ 'Ignores all files.') ++ return parser ++ ++ ++# Hardcoded dummy redirect URI for non-web apps. ++REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob' ++ ++ ++def UrlEscape(text): ++ # See OAUTH 5.1 for a definition of which characters need to be escaped. ++ return urlparse.quote(text, safe='~-._') ++ ++ ++def UrlUnescape(text): ++ # See OAUTH 5.1 for a definition of which characters need to be escaped. ++ return urlparse.unquote(text) ++ ++ ++def FormatUrlParams(params): ++ """Formats parameters into a URL query string. ++ ++ Args: ++ params: A key-value map. ++ ++ Returns: ++ A URL query string version of the given parameters. ++ """ ++ param_fragments = [] ++ for param in sorted(params.items(), key=lambda x: x[0]): ++ param_fragments.append('%s=%s' % (param[0], UrlEscape(param[1]))) ++ return '&'.join(param_fragments) ++ ++ ++def GeneratePermissionUrl(client_id, scope, auth_url): ++ """Generates the URL for authorizing access. ++ ++ This uses the "OAuth2 for Installed Applications" flow described at ++ https://developers.google.com/accounts/docs/OAuth2InstalledApp ++ ++ Args: ++ client_id: Client ID obtained by registering your app. ++ scope: scope for access token, e.g. 'https://mail.google.com' ++ Returns: ++ A URL that the user should visit in their browser. ++ """ ++ if not scope: ++ scope = 'https://mail.google.com/' ++ if not auth_url: ++ auth_url = 'https://accounts.google.com/o/oauth2/auth' ++ params = {} ++ params['client_id'] = client_id ++ params['redirect_uri'] = REDIRECT_URI ++ params['scope'] = scope ++ params['response_type'] = 'code' ++ return '%s?%s' % (auth_url, FormatUrlParams(params)) ++ ++ ++def AuthorizeTokens(client_id, client_secret, authorization_code, token_url): ++ """Obtains OAuth access token and refresh token. ++ ++ This uses the application portion of the "OAuth2 for Installed Applications" ++ flow at https://developers.google.com/accounts/docs/OAuth2InstalledApp#handlingtheresponse ++ ++ Args: ++ client_id: Client ID obtained by registering your app. ++ client_secret: Client secret obtained by registering your app. ++ authorization_code: code generated by Google Accounts after user grants ++ permission. ++ Returns: ++ The decoded response from the Google Accounts server, as a dict. Expected ++ fields include 'access_token', 'expires_in', and 'refresh_token'. ++ """ ++ params = {} ++ params['client_id'] = client_id ++ params['client_secret'] = client_secret ++ params['code'] = authorization_code ++ params['redirect_uri'] = REDIRECT_URI ++ params['grant_type'] = 'authorization_code' ++ if not token_url: ++ token_url = 'https://accounts.google.com/o/oauth2/token' ++ ++ response = urlopen.urlopen(token_url, ++ urlparse.urlencode(params).encode('ascii')).read() ++ return json.loads(response.decode("utf-8")) ++ ++ ++def RefreshToken(client_id, client_secret, refresh_token, token_url): ++ """Obtains a new token given a refresh token. ++ ++ See https://developers.google.com/accounts/docs/OAuth2InstalledApp#refresh ++ ++ Args: ++ client_id: Client ID obtained by registering your app. ++ client_secret: Client secret obtained by registering your app. ++ refresh_token: A previously-obtained refresh token. ++ Returns: ++ The decoded response from the Google Accounts server, as a dict. Expected ++ fields include 'access_token', 'expires_in', and 'refresh_token'. ++ """ ++ params = {} ++ params['client_id'] = client_id ++ params['client_secret'] = client_secret ++ params['refresh_token'] = refresh_token ++ params['grant_type'] = 'refresh_token' ++ if not token_url: ++ token_url = 'https://accounts.google.com/o/oauth2/token' ++ ++ response = urlopen.urlopen(token_url, ++ urlparse.urlencode(params).encode('ascii')).read() ++ return json.loads(response.decode("utf-8")) ++ ++ ++def GenerateOAuth2String(username, access_token, base64_encode=True): ++ """Generates an IMAP OAuth2 authentication string. ++ ++ See https://developers.google.com/google-apps/gmail/oauth2_overview ++ ++ Args: ++ username: the username (email address) of the account to authenticate ++ access_token: An OAuth2 access token. ++ base64_encode: Whether to base64-encode the output. ++ ++ Returns: ++ The SASL argument for the OAuth2 mechanism. ++ """ ++ auth_string = 'user=%s\1auth=Bearer %s\1\1' % (username, access_token) ++ if base64_encode: ++ auth_string = base64.b64encode(auth_string) ++ return auth_string ++ ++ ++def TestImapAuthentication(user, auth_string): ++ """Authenticates to IMAP with the given auth_string. ++ ++ Prints a debug trace of the attempted IMAP connection. ++ ++ Args: ++ user: The Gmail username (full email address) ++ auth_string: A valid OAuth2 string, as returned by GenerateOAuth2String. ++ Must not be base64-encoded, since imaplib does its own base64-encoding. ++ """ ++ print() ++ imap_conn = imaplib.IMAP4_SSL('imap.gmail.com') ++ imap_conn.debug = 4 ++ imap_conn.authenticate('XOAUTH2', lambda x: auth_string) ++ imap_conn.select('INBOX') ++ ++ ++def TestSmtpAuthentication(user, auth_string): ++ """Authenticates to SMTP with the given auth_string. ++ ++ Args: ++ user: The Gmail username (full email address) ++ auth_string: A valid OAuth2 string, not base64-encoded, as returned by ++ GenerateOAuth2String. ++ """ ++ print() ++ smtp_conn = smtplib.SMTP('smtp.gmail.com', 587) ++ smtp_conn.set_debuglevel(True) ++ smtp_conn.ehlo('test') ++ smtp_conn.starttls() ++ smtp_conn.docmd('AUTH', 'XOAUTH2 ' + base64.b64encode(auth_string)) ++ ++ ++def RequireOptions(options, *args): ++ missing = [arg for arg in args if getattr(options, arg) is None] ++ if missing: ++ print('Missing options: %s' % ' '.join(missing)) ++ sys.exit(-1) ++ ++def parseConfigFile(options): ++ if options.config_file: ++ cfg = dict(line.strip().split('=',1) for line in open(options.config_file)) ++ else: ++ cfg = { } ++ # defaults: ++ if not 'scope' in cfg: ++ cfg['scope'] = 'https://mail.google.com/' ++ if not 'max_age_sec' in cfg: ++ cfg['max_age_sec'] = '3000' ++ if not 'umask' in cfg: ++ cfg['umask'] = '0077' ++ if not 'auth_url' in cfg: ++ cfg['auth_url'] = 'https://accounts.google.com/o/oauth2/auth' ++ if not 'token_url' in cfg: ++ cfg['token_url'] = 'https://accounts.google.com/o/oauth2/token' ++ # overrides (from command line): ++ for arg in [ 'scope', 'client_id', 'client_secret', 'umask', ++ 'max_age_sec', 'access_token_file', 'refresh_token_file', ++ 'auth_url', 'token_url' ]: ++ if getattr(options,arg): ++ cfg[arg] = getattr(options,arg) ++ return cfg ++ ++def requireConfig(cfg, *args): ++ missing = [arg for arg in args if not arg in cfg] ++ if missing: ++ print('Missing options: %s' % ' '.join(missing)) ++ sys.exit(-1) ++ ++ ++def main(argv): ++ options_parser = SetupOptionParser() ++ (options, args) = options_parser.parse_args() ++ if options.refresh: ++ cfg = parseConfigFile(options) ++ requireConfig(cfg, 'refresh_token_file', 'access_token_file', ++ 'client_id', 'client_secret', 'umask') ++ st = os.stat(cfg['access_token_file']) ++ if options.refresh < 2: ++ requireConfig(cfg, 'max_age_sec') ++ if time.time()-st.st_mtime < int(cfg['max_age_sec']): ++ return ++ with open(cfg['refresh_token_file'],"r") as f: ++ reftok = f.readline().rstrip() ++ if len(reftok) == 0: ++ print('refresh token is empty') ++ sys.exit(-1) ++ response = RefreshToken(cfg['client_id'],cfg['client_secret'],reftok, ++ cfg['token_url']) ++ newTok = response['access_token'] ++ if len(newTok) == 0: ++ print('failed to obtain access token: it is empty') ++ sys.exit(-1) ++ savedUmask = os.umask(int(cfg['umask'],8)) ++ try: ++ with open(cfg['access_token_file']+".tmp","w") as f: ++ f.write(newTok) ++ f.write('\n') ++ os.rename(cfg['access_token_file']+".tmp",cfg['access_token_file']) ++ finally: ++ os.umask(savedUmask) ++ elif options.obtain_refresh_token_file: ++ cfg = parseConfigFile(options) ++ requireConfig(cfg, 'refresh_token_file', 'access_token_file', ++ 'client_id', 'client_secret', 'umask') ++ print('To authorize token, visit this url and follow the directions:') ++ print(' %s' % GeneratePermissionUrl(cfg['client_id'], cfg['scope'], ++ cfg['auth_url'])) ++ authorization_code = input('Enter verification code: ') ++ response = AuthorizeTokens(cfg['client_id'], cfg['client_secret'], ++ authorization_code, cfg['token_url']) ++ newRefTok = response['refresh_token'] ++ if len(newRefTok) == 0: ++ print('failed to obtain refresh token: it is empty') ++ sys.exit(-1) ++ newTok = response['access_token'] ++ if len(newTok) == 0: ++ print('failed to obtain corresponding access token: it is empty') ++ sys.exit(-1) ++ savedUmask = os.umask(int(cfg['umask'],8)) ++ try: ++ with open(cfg['refresh_token_file']+".tmp","w") as f: ++ f.write(newRefTok) ++ f.write('\n') ++ os.rename(cfg['refresh_token_file']+".tmp",cfg['refresh_token_file']) ++ with open(cfg['access_token_file']+".tmp","w") as f: ++ f.write(newTok) ++ f.write('\n') ++ print("Refresh token saved to '%s'" % cfg['refresh_token_file']) ++ print("Initial access token saved to '%s'" % cfg['access_token_file']) ++ print('Access Token Expiration Seconds: %s' % response['expires_in']) ++ os.rename(cfg['access_token_file']+".tmp",cfg['access_token_file']) ++ finally: ++ os.umask(savedUmask) ++ ++ ##### (OLD/testing options) ++ ++ elif options.refresh_token: ++ RequireOptions(options, 'client_id', 'client_secret') ++ response = RefreshToken(options.client_id, options.client_secret, ++ options.refresh_token, options.token_url) ++ print('Access Token: %s' % response['access_token']) ++ print('Access Token Expiration Seconds: %s' % response['expires_in']) ++ elif options.generate_oauth2_string: ++ RequireOptions(options, 'user', 'access_token') ++ print ('OAuth2 argument:\n' + ++ GenerateOAuth2String(options.user, options.access_token)) ++ elif options.generate_oauth2_token: ++ RequireOptions(options, 'client_id', 'client_secret') ++ print('To authorize token, visit this url and follow the directions:') ++ print(' %s' % GeneratePermissionUrl(options.client_id, options.scope, ++ options.auth_url)) ++ authorization_code = input('Enter verification code: ') ++ response = AuthorizeTokens(options.client_id, options.client_secret, ++ authorization_code, options.token_url) ++ print('Refresh Token: %s' % response['refresh_token']) ++ print('Access Token: %s' % response['access_token']) ++ print('Access Token Expiration Seconds: %s' % response['expires_in']) ++ elif options.test_imap_authentication: ++ RequireOptions(options, 'user', 'access_token') ++ TestImapAuthentication(options.user, ++ GenerateOAuth2String(options.user, options.access_token, ++ base64_encode=False)) ++ elif options.test_smtp_authentication: ++ RequireOptions(options, 'user', 'access_token') ++ TestSmtpAuthentication(options.user, ++ GenerateOAuth2String(options.user, options.access_token, ++ base64_encode=False)) ++ else: ++ options_parser.print_help() ++ print('Nothing to do, exiting.') ++ return ++ ++ ++if __name__ == '__main__': ++ main(sys.argv) +--- a/fetchmail.man ++++ b/fetchmail.man +@@ -1062,7 +1062,8 @@ External tools are necessary to obtain + such tokens. Access tokens often expire fairly quickly (e.g. 1 hour), + and new ones need to be generated from renewal tokens, so the + "passwordfile", "passwordfd", or "pwmd_*" options may be useful. See the +-oauth2.py script from ++contrib/fetchmail-oauth2.py script from the fetchmail source code, which ++was derived from code associated with + .URL https://github.com/google/gmail-oauth2-tools/wiki/OAuth2DotPyRunThrough "Google's Oauth2 Run Through" , + and other oauth2 documentation. For services like gmail, an "App Password" + is probably preferable if available, because it has roughly the same diff --git a/fetchmail-add-imap-oauthbearer-support.patch b/fetchmail-add-imap-oauthbearer-support.patch new file mode 100644 index 0000000..b09e56a --- /dev/null +++ b/fetchmail-add-imap-oauthbearer-support.patch @@ -0,0 +1,279 @@ +From: Matthew Ogilvie +Date: Sat, 27 May 2017 15:32:28 -0600 +Subject: add imap oauthbearer support +Git-repo: https://gitlab.com/fetchmail/fetchmail.git +Git-commit: 5c44df6df70b90f06d3204c6fbdd1ff19e990ca0 + +This expects an oauth2 access token in place of password. +When configured, it will also fall back on trying xoauth2. +--- + conf.c | 2 + + fetchmail.c | 3 + + fetchmail.h | 2 + + fetchmail.man | 23 +++++++++++- + fetchmailconf.py | 2 - + imap.c | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ + options.c | 2 + + rcfile_l.l | 1 + 8 files changed, 136 insertions(+), 3 deletions(-) + +--- a/conf.c ++++ b/conf.c +@@ -288,6 +288,8 @@ void dump_config(struct runctl *runp, st + stringdump("auth", "otp"); + else if (ctl->server.authenticate == A_MSN) + stringdump("auth", "msn"); ++ else if (ctl->server.authenticate == A_OAUTHBEARER) ++ stringdump("auth", "oauthbearer"); + + #ifdef HAVE_RES_SEARCH + booldump("dns", ctl->server.dns); +--- a/fetchmail.c ++++ b/fetchmail.c +@@ -1766,6 +1766,9 @@ static void dump_params (struct runctl * + case A_SSH: + printf(GT_(" End-to-end encryption assumed.\n")); + break; ++ case A_OAUTHBEARER: ++ printf(GT_(" OAUTHBEARER will be forced; expecting password to really be OAUTH2 authentication token\n")); ++ break; + } + if (ctl->server.principal != (char *) NULL) + printf(GT_(" Mail service principal is: %s\n"), ctl->server.principal); +--- a/fetchmail.h ++++ b/fetchmail.h +@@ -79,6 +79,7 @@ struct addrinfo; + #define A_SSH 8 /* authentication at session level */ + #define A_MSN 9 /* same as NTLM with keyword MSN */ + #define A_EXTERNAL 10 /* external authentication (client cert) */ ++#define A_OAUTHBEARER 11 /** oauth2 access token (not password) */ + + /* some protocols or authentication types (KERBEROS, GSSAPI, SSH) don't + * require a password */ +@@ -114,6 +115,7 @@ struct addrinfo; + #define MSGBUFSIZE 8192 + + #define NAMELEN 64 /* max username length */ ++/* oauth2 access tokens seem to be about 130 characters; make this longer: */ + #define PASSWORDLEN 256 /* max password length */ + #define DIGESTLEN 33 /* length of MD5 digest */ + +--- a/fetchmail.man ++++ b/fetchmail.man +@@ -1001,7 +1001,7 @@ AUTHENTICATION below for details). The + \&\fBpassword\fP, \fBkerberos_v5\fP, \fBkerberos\fP (or, for + excruciating exactness, \fBkerberos_v4\fP), \fBgssapi\fP, + \fBcram\-md5\fP, \fBotp\fP, \fBntlm\fP, \fBmsn\fP (only for POP3), +-\fBexternal\fP (only IMAP) and \fBssh\fP. ++\fBexternal\fP (only IMAP), \fBssh\fP and \fBoauthbearer\fP (only IMAP). + When \fBany\fP (the default) is specified, fetchmail tries + first methods that don't require a password (EXTERNAL, GSSAPI, KERBEROS\ IV, + KERBEROS\ 5); then it looks for methods that mask your password +@@ -1021,6 +1021,23 @@ GSSAPI or K4. Choosing KPOP protocol au + authentication. This option does not work with ETRN. GSSAPI service names are + in line with RFC-2743 and IANA registrations, see + .URL https://www.iana.org/assignments/gssapi-service-names/ "Generic Security Service Application Program Interface (GSSAPI)/Kerberos/Simple Authentication and Security Layer (SASL) Service Names" . ++.sp ++\fBoauthbearer\fP expects the supplied password to be an oauth2 authentication ++token instead of a password, as used by services like gmail. ++See RFC 7628 and RFC 6750. The \fBoauthbearer\fP ++setting also allows the non-standard "xoauth2" SASL scheme (using ++the same token) if the server only claims to support "xoauth2". ++External tools are necessary to obtain ++such tokens. Access tokens often expire fairly quickly (e.g. 1 hour), ++and new ones need to be generated from renewal tokens. See the ++oauth2.py script from ++.URL https://github.com/google/gmail-oauth2-tools/wiki/OAuth2DotPyRunThrough "Google's Oauth2 Run Through" , ++and other oauth2 documentation. For services like gmail, an "App Password" ++is probably preferable if available, because it has roughly the same ++security risks, and is a whole lot simpler to get working. "App Password" ++and oauthbearer both need to protect secrets on the client machine (files) and ++over the network (SSL/TLS). But "App Password" is ++sometimes completely disabled by business "G-suite" administrators. + .SS Miscellaneous Options + .TP + .B \-f | \-\-fetchmailrc +@@ -2327,7 +2344,9 @@ Legal protocol identifiers for use with + .PP + Legal authentication types are 'any', 'password', 'kerberos', + \&'kerberos_v4', 'kerberos_v5' and 'gssapi', 'cram\-md5', 'otp', 'msn' +-(only for POP3), 'ntlm', 'ssh', 'external' (only IMAP). ++(only for POP3), 'ntlm', 'ssh', 'external' (only IMAP), ++'oauthbearer' (only for IMAP; requires authentication token in ++place of password). + The 'password' type specifies + authentication by normal transmission of a password (the password may be + plain text or subject to protocol-specific encryption as in CRAM-MD5); +--- a/fetchmailconf.py ++++ b/fetchmailconf.py +@@ -487,7 +487,7 @@ defaultports = {"auto":None, + "ODMR":"odmr"} + + authlist = ("any", "password", "gssapi", "kerberos", "ssh", "otp", +- "msn", "ntlm") ++ "msn", "ntlm", "oauthbearer") + + listboxhelp = { + 'title' : 'List Selection Help', +--- a/imap.c ++++ b/imap.c +@@ -26,6 +26,10 @@ + #define IMAP4 0 /* IMAP4 rev 0, RFC1730 */ + #define IMAP4rev1 1 /* IMAP4 rev 1, RFC2060 */ + ++/* imap_plus_cont_context values */ ++#define IPLUS_NONE 0 ++#define IPLUS_OAUTHBEARER 1 /* oauthbearer (for more error info) */ ++ + /* global variables: please reinitialize them explicitly for proper + * working in daemon mode */ + +@@ -38,6 +42,8 @@ static int imap_version = IMAP4; + static flag do_idle = FALSE, has_idle = FALSE; + static int expunge_period = 1; + ++static int plus_cont_context = IPLUS_NONE; ++ + /* mailbox variables initialized in imap_getrange() */ + static int count = 0, oldcount = 0, recentcount = 0, unseen = 0, deletions = 0; + static unsigned int startcount = 1; +@@ -202,6 +208,21 @@ static int imap_response(int sock, char + if (ok != PS_SUCCESS) + return(ok); + ++ if (buf[0] == '+' && buf[1] == ' ') { ++ if (plus_cont_context == IPLUS_OAUTHBEARER) { ++ /* future: Consider decoding the base64-encoded JSON ++ * error response info and logging it. But for now, ++ * ignore continuation data, send the expected blank ++ * line, and assume that the next response will be ++ * a tagged "NO" as documented. ++ */ ++ SockWrite(sock, "\r\n", 2); ++ if (outlevel >= O_MONITOR) ++ report(stdout, "IMAP> \n"); ++ continue; ++ } ++ } ++ + /* all tokens in responses are caseblind */ + for (cp = buf; *cp; cp++) + if (islower((unsigned char)*cp)) +@@ -316,6 +337,69 @@ static int do_imap_ntlm(int sock, struct + } + #endif /* NTLM */ + ++static int do_imap_oauthbearer(int sock, struct query *ctl,flag xoauth2) ++{ ++ /* Implements relevant parts of RFC-7628, RFC-6750, and ++ * https://developers.google.com/gmail/imap/xoauth2-protocol ++ * ++ * This assumes something external manages obtaining an up-to-date ++ * authentication/bearer token and arranging for it to be in ++ * ctl->password. This may involve renewing it ahead of time if ++ * necessary using a renewal token that fetchmail knows nothing about. ++ * See: ++ * https://github.com/google/gmail-oauth2-tools/wiki/OAuth2DotPyRunThrough ++ */ ++ const char *name; ++ char *oauth2str; ++ int oauth2len; ++ int saved_suppress_tags = suppress_tags; ++ ++ char *oauth2b64; ++ ++ int ok; ++ ++ oauth2len = strlen(ctl->remotename) + strlen(ctl->password) + 32; ++ oauth2str = (char *)xmalloc(oauth2len); ++ if (xoauth2) ++ { ++ snprintf(oauth2str, oauth2len, ++ "user=%s\1auth=Bearer %s\1\1", ++ ctl->remotename, ++ ctl->password); ++ name = "XOAUTH2"; ++ } ++ else ++ { ++ snprintf(oauth2str, oauth2len, ++ "n,a=%s,\1auth=Bearer %s\1\1", ++ ctl->remotename, ++ ctl->password); ++ name = "OAUTHBEARER"; ++ } ++ ++ oauth2b64 = (char *)xmalloc(2*strlen(oauth2str)+8); ++ to64frombits(oauth2b64, oauth2str, strlen(oauth2str)); ++ ++ memset(oauth2str, 0x55, strlen(oauth2str)); ++ free(oauth2str); ++ ++ /* Protect the access token like a password in logs, despite the ++ * usually-short expiration time and base64 encoding: ++ */ ++ strlcpy(shroud, oauth2b64, sizeof(shroud)); ++ ++ plus_cont_context = IPLUS_OAUTHBEARER; ++ ok = gen_transact(sock, "AUTHENTICATE %s %s", name, oauth2b64); ++ plus_cont_context = IPLUS_NONE; ++ ++ memset(shroud, 0x55, sizeof(shroud)); ++ shroud[0] = '\0'; ++ memset(oauth2b64, 0x55, strlen(oauth2b64)); ++ free(oauth2b64); ++ ++ return ok; ++} ++ + static void imap_canonicalize(char *result, char *raw, size_t maxlen) + /* encode an IMAP password as per RFC1730's quoting conventions */ + { +@@ -510,6 +594,26 @@ static int imap_getauth(int sock, struct + */ + ok = PS_AUTHFAIL; + ++ if (ctl->server.authenticate == A_OAUTHBEARER) ++ { ++ /* Fetchmail's oauthbearer and xoauth2 support expects the "password" ++ * to actually be an oauth2 authentication token, so only ++ * try these options if specifically enabled. ++ * (Generating a token using the complex https-based oauth2 ++ * protocol is left as an exercise for the user.) ++ */ ++ if (strstr(capabilities, "AUTH=OAUTHBEARER") || ++ !strstr(capabilities, "AUTH=XOAUTH2")) ++ { ++ ok = do_imap_oauthbearer(sock, ctl, FALSE); /* OAUTHBEARER */ ++ } ++ if (ok && strstr(capabilities, "AUTH=XOAUTH2")) ++ { ++ ok = do_imap_oauthbearer(sock, ctl, TRUE); /* XOAUTH2 */ ++ } ++ return ok; ++ } ++ + /* Yahoo hack - we'll just try ID if it was offered by the server, + * and IGNORE errors. */ + { +--- a/options.c ++++ b/options.c +@@ -421,6 +421,8 @@ int parsecmdline (int argc /** argument + ctl->server.authenticate = A_ANY; + else if (strcmp(optarg, "msn") == 0) + ctl->server.authenticate = A_MSN; ++ else if (strcmp(optarg, "oauthbearer") == 0) ++ ctl->server.authenticate = A_OAUTHBEARER; + else { + fprintf(stderr,GT_("Invalid authentication `%s' specified.\n"), optarg); + errflag++; +--- a/rcfile_l.l ++++ b/rcfile_l.l +@@ -106,6 +106,7 @@ cram(-md5)? { SETSTATE(0); yylval.proto + msn { SETSTATE(0); yylval.proto = A_MSN; return AUTHTYPE;} + ntlm { SETSTATE(0); yylval.proto = A_NTLM; return AUTHTYPE;} + password { SETSTATE(0); yylval.proto = A_PASSWORD; return AUTHTYPE;} ++oauthbearer { SETSTATE(0); yylval.proto = A_OAUTHBEARER; return AUTHTYPE;} + timeout { return TIMEOUT;} + envelope { return ENVELOPE; } + qvirtual { return QVIRTUAL; } diff --git a/fetchmail-add-passwordfile-and-passwordfd-options.patch b/fetchmail-add-passwordfile-and-passwordfd-options.patch new file mode 100644 index 0000000..0753881 --- /dev/null +++ b/fetchmail-add-passwordfile-and-passwordfd-options.patch @@ -0,0 +1,317 @@ +From: Matthew Ogilvie +Date: Sun, 28 May 2017 00:01:02 -0600 +Subject: add passwordfile and passwordfd options +Git-repo: https://gitlab.com/fetchmail/fetchmail.git +Git-commit: cdd7182f65734c97723ba5f282040e08d830e650 + +--- + fetchmail.c | 82 +++++++++++++++++++++++++++++++++++++++++++++++++-- + fetchmail.h | 2 ++ + fetchmail.man | 40 ++++++++++++++++++++++++- + options.c | 16 ++++++++++ + rcfile_l.l | 2 ++ + rcfile_y.y | 6 ++++ + 6 files changed, 145 insertions(+), 3 deletions(-) + +diff --git a/fetchmail.c b/fetchmail.c +index ead6d1f2..0292d42a 100644 +--- a/fetchmail.c ++++ b/fetchmail.c +@@ -387,7 +387,7 @@ int main(int argc, char **argv) + /* Server won't care what the password is, but there + must be some non-null string here. */ + ctl->password = ctl->remotename; +- else ++ else if (!ctl->passwordfile && ctl->passwordfd==-1) + { + netrc_entry *p; + +@@ -554,8 +554,81 @@ int main(int argc, char **argv) + if (ctl->active && !(implicitmode && ctl->server.skip) + && !NO_PASSWORD(ctl) && !ctl->password) + { +- if (!isatty(0)) ++ if (ctl->passwordfd != -1) + { ++ char msg[PASSWORDLEN+1]; ++ char *mi; ++ ++ /* Read one character at a time to avoid reading too ++ * much if more than one password sent in through this FD ++ * (although that would be a questionable practice). ++ */ ++ for (mi = msg; mipasswordfd, mi, 1); ++ if(res == -1) { ++ int saveErrno = errno; ++ fprintf(stderr, ++ GT_("fetchmail: unable to read password from fd=%d: %s\n"), ++ ctl->passwordfd, ++ strerror(saveErrno)); ++ memset(msg, 0x55, mi-msg); ++ return PS_AUTHFAIL; ++ } ++ if (res == 0 || *mi == '\n') ++ break; ++ } ++ *mi = '\0'; ++ if (mi == msg) { ++ fprintf(stderr, ++ GT_("fetchmail: empty password read from fd=%d\n"), ++ ctl->passwordfd); ++ return PS_AUTHFAIL; ++ } ++ ++ ctl->password = xstrdup(msg); ++ memset(msg, 0x55, mi-msg); ++ } else if (ctl->passwordfile) { ++ int fd = open(ctl->passwordfile, O_RDONLY); ++ char msg[PASSWORDLEN+1]; ++ char *newline; ++ int res; ++ ++ if (fd == -1) { ++ int saveErrno = errno; ++ fprintf(stderr, ++ GT_("fetchmail: unable to open %s: %s\n"), ++ ctl->passwordfile, ++ strerror(saveErrno)); ++ return PS_AUTHFAIL; ++ } ++ ++ res = read(fd, msg, sizeof(msg)-1); ++ if (res == -1 || close(fd) == -1) { ++ int saveErrno = errno; ++ fprintf(stderr, ++ GT_("fetchmail: error reading %s: %s\n"), ++ ctl->passwordfile, ++ strerror(saveErrno)); ++ return PS_AUTHFAIL; ++ } ++ msg[res] = '\0'; ++ ++ newline = memchr(msg, '\n', res); ++ if (newline != NULL) { ++ *newline = '\0'; ++ } ++ ++ if (strlen(msg) == 0) { ++ fprintf(stderr, ++ GT_("fetchmail: empty password read from %s\n"), ++ ctl->passwordfile); ++ memset(msg, 0x55, res); ++ return PS_AUTHFAIL; ++ } ++ ++ ctl->password = xstrdup(msg); ++ memset(msg, 0x55, res); ++ } else if (!isatty(0)) { + fprintf(stderr, + GT_("fetchmail: can't find a password for %s@%s.\n"), + ctl->remotename, ctl->server.pollname); +@@ -1000,6 +1073,10 @@ static void optmerge(struct query *h2, struct query *h1, int force) + FLAG_MERGE(wildcard); + STRING_MERGE(remotename); + STRING_MERGE(password); ++ FLAG_MERGE(passwordfile); ++ if (force ? h1->passwordfd!=-1 : h2->passwordfd==-1) { ++ h2->passwordfd = h1->passwordfd; ++ } + STRING_MERGE(mda); + STRING_MERGE(bsmtp); + FLAG_MERGE(listener); +@@ -1064,6 +1141,7 @@ static int load_params(int argc, char **argv, int optind) + def_opts.smtp_socket = -1; + def_opts.smtpaddress = (char *)0; + def_opts.smtpname = (char *)0; ++ def_opts.passwordfd = -1; + def_opts.server.protocol = P_AUTO; + def_opts.server.timeout = CLIENT_TIMEOUT; + def_opts.server.esmtp_name = user; +diff --git a/fetchmail.h b/fetchmail.h +index 22b72827..715cc2d0 100644 +--- a/fetchmail.h ++++ b/fetchmail.h +@@ -336,6 +336,8 @@ struct query + int wildcard; /* should unmatched names be passed through */ + char *remotename; /* remote login name to use */ + char *password; /* remote password to use */ ++ char *passwordfile; /* filename; first line contains password */ ++ int passwordfd; /* fileno that password will be piped into */ + struct idlist *mailboxes; /* list of mailboxes to check */ + + /* per-forwarding-target data */ +diff --git a/fetchmail.man b/fetchmail.man +index aece716e..9c6ed4ad 100644 +--- a/fetchmail.man ++++ b/fetchmail.man +@@ -872,6 +872,37 @@ The default is your login name on the client machine that is running + \fBfetchmail\fP. + See USER AUTHENTICATION below for a complete description. + .TP ++.B \-\-passwordfile ++(Keyword: passwordfile) ++.br ++Specifies a file name from which to read the first line to use as the password. ++Useful if something changes the password/token often without regenerating a ++long fetchmailrc file, such as with typical xoauth2 authentication tokens. ++Protect the file with appropriate permissions to avoid leaking your password. ++Fetchmail might not re-read the file in daemon mode (-d) unless the ++fetchmailrc file also changes, so it might make sense to run it in ++non-daemon mode from some other background process (cron and/or whatever ++updates the password). ++.TP ++.B \-\-passwordfd ++(Keyword: passwordfd) ++.br ++Specifies a file descriptor number inherited from the calling process, ++from which fetchmail should read one line to use as the password. ++The descriptor will usually be the receiving end of a pipe (equivalent ++to "something | fetchmail \-\-passwordfd 5 5<\&0"), ++although it could also be a redirected input file ++(equivalent to "fetchmail \-\-passwordfd 5 5 | \-\-interface + (Keyword: interface) + .br +@@ -955,7 +986,8 @@ setting also allows the non-standard "xoauth2" SASL scheme (using + the same token) if the server only claims to support "xoauth2". + External tools are necessary to obtain + such tokens. Access tokens often expire fairly quickly (e.g. 1 hour), +-and new ones need to be generated from renewal tokens. See the ++and new ones need to be generated from renewal tokens, so the ++"passwordfile", "passwordfd", or "pwmd_*" options may be useful. See the + oauth2.py script from + .URL https://github.com/google/gmail-oauth2-tools/wiki/OAuth2DotPyRunThrough "Google's Oauth2 Run Through" , + and other oauth2 documentation. For services like gmail, an "App Password" +@@ -1844,6 +1876,12 @@ T} + pass[word] \& \& T{ + Specify remote account password + T} ++passwordfile \-\-... \& T{ ++File name with password in first line. ++T} ++passwordfd \-\-... \& T{ ++Inherited file descriptor from which to read one line for the password. ++T} + ssl \& \& T{ + Connect to server over the specified base protocol using SSL encryption + T} +diff --git a/options.c b/options.c +index a181c6d9..99b8e020 100644 +--- a/options.c ++++ b/options.c +@@ -26,6 +26,8 @@ enum { + LA_POSTMASTER, + LA_NOBOUNCE, + LA_AUTH, ++ LA_PASSWORDFILE, ++ LA_PASSWORDFD, + LA_FETCHDOMAINS, + LA_BSMTP, + LA_LMTP, +@@ -103,6 +105,8 @@ static const struct option longoptions[] = { + {"port", required_argument, (int *) 0, 'P' }, + {"service", required_argument, (int *) 0, 'P' }, + {"auth", required_argument, (int *) 0, LA_AUTH}, ++ {"passwordfile", required_argument, (int *) 0, LA_PASSWORDFILE }, ++ {"passwordfd", required_argument, (int *) 0, LA_PASSWORDFD }, + {"timeout", required_argument, (int *) 0, 't' }, + {"envelope", required_argument, (int *) 0, 'E' }, + {"qvirtual", required_argument, (int *) 0, 'Q' }, +@@ -236,6 +240,7 @@ int parsecmdline (int argc /** argument count */, + + memset(ctl, '\0', sizeof(struct query)); /* start clean */ + ctl->smtp_socket = -1; ++ ctl->passwordfd = -1; + + while (!errflag && + (c = getopt_long(argc,argv,shortoptions, +@@ -407,6 +412,17 @@ int parsecmdline (int argc /** argument count */, + errflag++; + } + break; ++ case LA_PASSWORDFILE: ++ ctl->passwordfile = optarg; ++ break; ++ case LA_PASSWORDFD: ++ ctl->passwordfd = xatoi(optarg, &errflag); ++ if (ctl->passwordfd < 0) { ++ fprintf(stderr,GT_("Invalid file descriptor %d\n"), ++ ctl->passwordfd); ++ errflag++; ++ } ++ break; + case 't': + ctl->server.timeout = xatoi(optarg, &errflag); + if (ctl->server.timeout == 0) +diff --git a/rcfile_l.l b/rcfile_l.l +index 824845b3..47a37d0f 100644 +--- a/rcfile_l.l ++++ b/rcfile_l.l +@@ -115,6 +115,8 @@ pwmd_socket { return PWMD_SOCKET; } + reject { return REJECT_; } + + user(name)? {SETSTATE(NAME); return USERNAME; } ++passwordfile { return PASSWORDFILE; } ++passwordfd { return PASSWORDFD; } + pass(word)? {SETSTATE(NAME); return PASSWORD; } + folder(s)? { return FOLDER; } + smtp(host)? { return SMTPHOST; } +diff --git a/rcfile_y.y b/rcfile_y.y +index 815fbd74..3967adb3 100644 +--- a/rcfile_y.y ++++ b/rcfile_y.y +@@ -61,6 +61,7 @@ extern char * yytext; + %token DEFAULTS POLL SKIP VIA AKA LOCALDOMAINS PROTOCOL + %token AUTHENTICATE TIMEOUT KPOP SDPS ENVELOPE QVIRTUAL + %token USERNAME PASSWORD FOLDER SMTPHOST FETCHDOMAINS MDA BSMTP LMTP ++%token PASSWORDFILE PASSWORDFD + %token SMTPADDRESS SMTPNAME SPAMRESPONSE PRECONNECT POSTCONNECT LIMIT WARNINGS + %token INTERFACE MONITOR PLUGIN PLUGOUT + %token IS HERE THERE TO MAP +@@ -315,6 +316,8 @@ user_option : TO mapping_list HERE + + | IS STRING THERE {current.remotename = $2;} + | PASSWORD STRING {current.password = $2;} ++ | PASSWORDFILE STRING {current.passwordfile = $2;} ++ | PASSWORDFD NUMBER {current.passwordfd = NUM_VALUE_IN($2);} + | FOLDER folder_list + | SMTPHOST smtp_list + | FETCHDOMAINS fetch_list +@@ -539,6 +542,7 @@ static void reset_server(const char *name, int skip) + trailer = FALSE; + memset(¤t,'\0',sizeof(current)); + current.smtp_socket = -1; ++ current.passwordfd = -1; + current.server.pollname = xstrdup(name); + current.server.skip = skip; + current.server.principal = (char *)NULL; +@@ -560,6 +564,7 @@ static void user_reset(void) + + memset(¤t, '\0', sizeof(current)); + current.smtp_socket = -1; ++ current.passwordfd = -1; + + current.server = save; + } +@@ -580,6 +585,7 @@ struct query *hostalloc(struct query *init /** pointer to block containing + { + memset(node, '\0', sizeof(struct query)); + node->smtp_socket = -1; ++ node->passwordfd = -1; + } + + /* append to end of list */ +-- +2.31.1 + diff --git a/fetchmail-add-query_to64_outsize-utility-function.patch b/fetchmail-add-query_to64_outsize-utility-function.patch new file mode 100644 index 0000000..240bcc3 --- /dev/null +++ b/fetchmail-add-query_to64_outsize-utility-function.patch @@ -0,0 +1,42 @@ +From: Matthew Ogilvie +Date: Fri, 21 Dec 2018 09:00:46 -0700 +Subject: add query_to64_outsize() utility function +Git-repo: https://gitlab.com/fetchmail/fetchmail.git +Git-commit: cc6e146d516140df800da68976eb7c0aa1cef7c0 + +--- + base64.c | 7 +++++++ + fetchmail.h | 1 + + 2 files changed, 8 insertions(+) + +diff --git a/base64.c b/base64.c +index 3cd41691..25393b35 100644 +--- a/base64.c ++++ b/base64.c +@@ -61,6 +61,13 @@ fail: + return rc; + } + ++size_t query_to64_outsize(size_t inlen) ++/* Returns how much space needs to be allocated to receive the output from ++ * to64frombits(), including the '\0' terminator. */ ++{ ++ return ((inlen+2)/3)*4+1; ++} ++ + int from64tobits(void *out_, const char *in, int maxlen) + /* base 64 to raw bytes in quasi-big-endian order, returning count of bytes */ + /* maxlen limits output buffer size, set to zero to ignore */ +diff --git a/fetchmail.h b/fetchmail.h +index 8b9dd6c4..2d378942 100644 +--- a/fetchmail.h ++++ b/fetchmail.h +@@ -638,6 +638,7 @@ int prc_filecheck(const char *, const flag); + + /* base64.c */ + int to64frombits(char *, const void *, int inlen, size_t outlen); ++size_t query_to64_outsize(size_t inlen); + int from64tobits(void *, const char *, int mxoutlen); + + /* unmime.c */ + diff --git a/fetchmail-add-readme-oauth2-issue-27.patch b/fetchmail-add-readme-oauth2-issue-27.patch new file mode 100644 index 0000000..b15611b --- /dev/null +++ b/fetchmail-add-readme-oauth2-issue-27.patch @@ -0,0 +1,164 @@ +From: William Bader +Date: Sun, 31 Jan 2021 06:42:46 +0000 +Subject: Add README.OAUTH2 issue #27 +Git-repo: https://gitlab.com/fetchmail/fetchmail.git +Git-commit: d52ba9652c9207358e0b9acc11403688f6f16b9e + +--- + README.OAUTH2 | 147 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 147 insertions(+) + +diff --git a/README.OAUTH2 b/README.OAUTH2 +new file mode 100644 +index 00000000..e861c812 +--- /dev/null ++++ b/README.OAUTH2 +@@ -0,0 +1,147 @@ ++OAUTH2 support for gmail ++======================== ++ ++Preface ++------- ++ ++fetchmail 7 adds support for OAuth2. ++You create a project in google that requests gmail access to request an OAuth2 client id and client secret. ++Then you use the contributed fetchmail-oauth2.py to request a refresh token for gmail access to your gmail account. ++Then you use the fetchmail-oauth2.py again to request temporary access tokens that fetchmail uses like a password. ++ ++Create a Google project and request an OAuth2 client id and client secret ++------------------------------------------------------------------------- ++ ++* Open the Google API Dashboard: https://console.developers.google.com/apis/dashboard ++* The first time that you enter the page, you will have to select your country and agree to terms of service. ++* You should see a title bar with "Google APIs" and a menu down the left with "Dashboard, Library, ++ Credentials, OAuth consent screen, Domain verification, Page usage agreements". ++* Click to create a new project, possibly on a pull-down arrow to the right of "Google APIs" on the title bar. ++* Click on "NEW PROJECT". ++ + Enter a project name like "fetchmail". ++ + You can leave "Location" as "No organization" for personal email. ++ + If you are a G Suite administrator, you might be able to enter your G Suite organization. ++ + When you enter the project name, you will get a message like "Project ID: fetchmail-123456. It cannot be changed later." ++ + Make a note of the full project name. ++ + Click on "CREATE". ++ + Google will take a few seconds to create the project. ++* Switch to the new project, either from "SELECT PROJECT" in the notification window or on the title bar pulldown after "Google APIs". ++* Click on "Credentials" on the menu at the left. ++* Click on "+ CREATE CREDENTIALS" at the top of the window. ++* Select "OAuth client ID" from the list of credential types. ++* Click on "CONFIGURE CONSENT SCREEN" at the right. ++ + Select "External" from the list of user types. "Internal" is for organizations with G Suite. ++ + Click "CREATE". ++* Fill out the app registration form. ++ + "App name" can be the full project name, like "fetchmail-123456". It has to be unique. ++ + "User support email" can be your gmail email. ++ + "App logo" can be empty. I used /usr/share/icons/Adwaita/256x256/legacy/emblem-mail.png ++ + "Application home page" can be empty. ++ + "Application privacy policy link" can be empty. ++ + "Application terms of service link" can be empty. ++ + "Authorized domain" can be empty. ++ + "Developer contact email address" can be your gmail email. ++ + Click "SAVE AND CONTINUE". ++ + Click "ADD OR REMOVE SCOPES" on the "Edit app registration" screen. ++ + Click on "Google API Library". This opens a new tab. ++ + Filter for "Email" and click on "Gmail API". ++ + Click on "ENABLE". ++ + Return to the "Edit app registration" tab and refresh. ++ + Click "ADD OR REMOVE SCOPES" on the "Edit app registration" screen. ++ + I think that the scope ".../auth/gmail.modify" to "View and modify but not delete your email" is sufficient. ++ + Click on "SAVE AND CONTINUE". This opens the "Test Users" window. ++ + Click on "+ ADD USERS". ++ + Enter you gmail address and click on "ADD". ++ + Click on "SAVE AND CONTINUE". ++ + This opens a "Summary" page. ++ + If you need to change something, click on "OAuth consent screen" on the menu at the left and then "EDIT APP" to step through the screens again. ++* Click on "Credentials" on the menu at the left to create client credentials. ++ + Click on "+ CREATE CREDENTIALS" at the top of the window. ++ + Select "OAuth client ID" from the list of credential types. ++ + Select "Desktop app" from the list of "Application types". ++ + "Name" can be "DesktopClient1" or whatever the screen suggests. ++ + Click on "CREATE". ++ + It will show a window with "Your Client ID" and "Your Client Secret". Copy them somewhere safe. ++ ++Download and build fetchmail 7 ++------------------------------ ++``` ++git clone https://gitlab.com/fetchmail/fetchmail.git ++cd fetchmail ++git checkout -t origin/next ++./autogen.sh ++./configure ++make ++make check ++sudo make install ++``` ++ ++Configure fetchmail-oauth2.py ++----------------------------- ++* Create a file, for example /home/yourname/.fetchmail-oauth2 ++``` ++client_id=YOUR-CLIENT-ID ++client_secret=YOUR-CLIENT-SECRET ++refresh_token_file=/home/yourname/.fetchmail-refresh ++access_token_file=/home/yourname/.fetchmail-token ++max_age_sec=3000 ++``` ++* Replace YOUR-CLIENT-ID and YOUR-CLIENT-SECRET with the keys for "Your Client ID" and "Your Client Secret" from the previous step. ++* The refresh and token files do not need to exist, but they have to be valid paths. ++* Run `contrib/fetchmail-oauth2.py -c /home/yourname/.fetchmail-oauth2 --obtain_refresh_token_file` ++ + The script will give you a URL. ++ + Paste the URL into a web browser. ++ + URL should open a google authentication page. ++ + Select the email account. ++ + Google will warn that the app isn't verified. Click on "Continue". ++ + Google will warn that "fetchmail-123456 wants to access your Google Account `your.name@gmail.com`". ++ + Click on "Allow". ++ + The page will display the sign in key. ++ + Paste the key into the script. ++ + The script will report: ++``` ++Refresh token saved to '/home/yourname/.fetchmail-refresh' ++Initial access token saved to '/home/yourname/.fetchmail-token' ++Access Token Expiration Seconds: 3599 ++``` ++ + Hopefully you will not need to do this again for months or years. ++* Run `chmod 0600` on all of the files .fetchmail-oauth2 .fetchmail-refresh .fetchmail-token ++ ++Configure fetchmail ++--------------------- ++* Create an entry in your `.fetchmailrc` ++``` ++poll imap.gmail.com protocol imap ++ auth oauthbearer username "your.name@gmail.com" ++ passwordfile "/home/yourname/.fetchmail-token" ++ is yourname here ++ fetchlimit 10 folder "Download" ++ keep ++ sslmode wrapped sslcertck ++``` ++* Run `chmod 0400` on your `.fetchmailrc` ++* The optional "fetchlimit #" limits the number of emails if you are testing. ++* The optional "folder name" sets the folder to check. ++* I made gmail filters that add a "Download" label to important emails. ++* Fetchmail downloads unread emails. You can go into gmail and mark a few emails unread for testing. ++* Try running fetchmail once at a command line. ++ ++Script fetchmail ++---------------- ++* Each access token expires after an hour. ++* If you run fetchmail from cron, you should run `fetchmail-oauth2.py -c /home/yourname/.fetchmail-oauth2 --auto_refresh ; fetchmail` ++* For example, `*/2 * * * * (fetchmail-oauth2.py -c /home/yourname/.fetchmail-oauth2 --auto_refresh ; fetchmail) > /home/yourname/fetchmail.log 2>&1` ++* The `--auto_refresh` option checks the age of the key against the `max_age_sec` and renews it if necessary. ++* `max_age_sec=3000` in `.fetchmail-oauth2` renews the key after 50 minutes, which should give a safe margin. ++ ++Further reading ++--------------- ++* Instructions by the author of OAuth2 support for Fetchmail and Postfix ++ + Setting Up OAUTH2 Support for Fetchmail and Postfix http://mmogilvi.users.sourceforge.net/software/oauthbearer.html ++ + Run `fetchmail-oauth2.py --help | less` ++* Documents from Google ++ + Using OAuth 2.0 to Access Google APIs https://developers.google.com/identity/protocols/oauth2 ++ + Integrating Google Sign-In into your web app https://developers.google.com/identity/sign-in/web/devconsole-project ++* Google links ++ + Google API Dashboard: https://console.developers.google.com/apis/dashboard ++ + Google Cloud Resource Manager: https://console.developers.google.com/cloud-resource-manager + diff --git a/fetchmail-bump-max-passwordlen-to-1bytes.patch b/fetchmail-bump-max-passwordlen-to-1bytes.patch new file mode 100644 index 0000000..ee5df76 --- /dev/null +++ b/fetchmail-bump-max-passwordlen-to-1bytes.patch @@ -0,0 +1,24 @@ +From: Matthias Andree +Date: Sat, 24 Apr 2021 15:12:01 +0200 +Subject: Bump max. passwordlen to 10000 bytes. +Git-repo: https://gitlab.com/fetchmail/fetchmail.git +Git-commit: 919fd787540c4a3fa4694566edce406df1e42001 + +--- + fetchmail.h | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/fetchmail.h b/fetchmail.h +index af4d0dd2..ac248805 100644 +--- a/fetchmail.h ++++ b/fetchmail.h +@@ -104,7 +104,7 @@ enum authenticators { + + #define NAMELEN 64 /* max username length */ + /* oauth2 access tokens seem to be about 130 characters; make this longer: */ +-#define PASSWORDLEN 4096 /* max password length; oauth2 tokens have no maximum length */ ++#define PASSWORDLEN 10000 /* max password length; oauth2 tokens have no maximum length */ + #define DIGESTLEN 33 /* length of MD5 digest */ + + /* exit code values */ + diff --git a/fetchmail-chase-and-integrate-interface-change.patch b/fetchmail-chase-and-integrate-interface-change.patch new file mode 100644 index 0000000..8cac8ec --- /dev/null +++ b/fetchmail-chase-and-integrate-interface-change.patch @@ -0,0 +1,32 @@ +From: Matthias Andree +Date: Sun, 25 Nov 2018 12:09:07 +0100 +Subject: Chase and integrate interface change. +Git-repo: https://gitlab.com/fetchmail/fetchmail.git +Git-commit: d52b7b6859d46134b46e6de9b408739b18745d47 + +--- + oauth2.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/oauth2.c b/oauth2.c +index a8a324b8..4f7a072a 100644 +--- a/oauth2.c ++++ b/oauth2.c +@@ -10,6 +10,7 @@ + #include "oauth2.h" + + #include ++#include + #include + + char *get_oauth2_string(struct query *ctl,flag xoauth2) +@@ -52,7 +53,7 @@ char *get_oauth2_string(struct query *ctl,flag xoauth2) + } + + oauth2b64 = (char *)xmalloc(2*strlen(oauth2str)+8); +- to64frombits(oauth2b64, oauth2str, strlen(oauth2str)); ++ to64frombits(oauth2b64, oauth2str, strlen(oauth2str), oauth2len); + + memset(oauth2str, 0x55, strlen(oauth2str)); + free(oauth2str); + diff --git a/fetchmail-give-each-ctl-it-s-own-copy-of-password.patch b/fetchmail-give-each-ctl-it-s-own-copy-of-password.patch new file mode 100644 index 0000000..89747ae --- /dev/null +++ b/fetchmail-give-each-ctl-it-s-own-copy-of-password.patch @@ -0,0 +1,48 @@ +From: Matthew Ogilvie +Date: Fri, 9 Jun 2017 19:31:17 -0600 +Subject: give each ctl it's own copy of password +Git-repo: https://gitlab.com/fetchmail/fetchmail.git +Git-commit: 469b0a212e7f047ab16ef46a9158df5fb373e8c2 + +pwdb_* and passwordfile options may free and re-allocate password +for each poll operation. Giving each context it's own copy of +the password should prevent accessing freed memory in another copy. + +I haven't tested pwmd, but these seem like obvious fixes. +--- + fetchmail.c | 12 ++++++++++-- + 1 file changed, 10 insertions(+), 2 deletions(-) + +diff --git a/fetchmail.c b/fetchmail.c +index 0292d42a..e2828a4f 100644 +--- a/fetchmail.c ++++ b/fetchmail.c +@@ -386,7 +386,7 @@ int main(int argc, char **argv) + if (NO_PASSWORD(ctl)) + /* Server won't care what the password is, but there + must be some non-null string here. */ +- ctl->password = ctl->remotename; ++ ctl->password = xstrdup(ctl->remotename); + else if (!ctl->passwordfile && ctl->passwordfd==-1) + { + netrc_entry *p; +@@ -1072,7 +1072,15 @@ static void optmerge(struct query *h2, struct query *h1, int force) + + FLAG_MERGE(wildcard); + STRING_MERGE(remotename); +- STRING_MERGE(password); ++ if (force ? !!h1->password : !h2->password) { ++ if (h2->password) { ++ memset(h2->password, 0x55, strlen(h2->password)); ++ xfree(h2->password); ++ } ++ if (h1->password) { ++ h2->password = xstrdup(h1->password); ++ } ++ } + FLAG_MERGE(passwordfile); + if (force ? h1->passwordfd!=-1 : h2->passwordfd==-1) { + h2->passwordfd = h1->passwordfd; +-- +2.31.1 + diff --git a/fetchmail-increase-max-password-length-to-handle-oauth-tokens.patch b/fetchmail-increase-max-password-length-to-handle-oauth-tokens.patch new file mode 100644 index 0000000..d6103f4 --- /dev/null +++ b/fetchmail-increase-max-password-length-to-handle-oauth-tokens.patch @@ -0,0 +1,33 @@ +From: =?utf-8?q?Martin_Sj=C3=B6lund_=3Cmartin=40sjoelund=2Ese=3E?= +Date: Thu, 17 Dec 2020 09:09:44 +0100 +Subject: Increase max password length to handle oauth tokens +Git-repo: https://gitlab.com/fetchmail/fetchmail.git +Git-commit: 6e877b5d92527ad501aaef46e37704b51db316fb + +The maximum length of oauth2 access tokens is unbounded. +* Google uses 2048 byte access tokens and "Google reserves the right + to change token size within these limits, and your application + must support variable token sizes accordingly." +* My Office365 access token is 2108 bytes long. +* Intuit says you must support 4096 byte access tokens. + +This simply patches the hard-coded limit to 4096 bytes, but it might +not be sufficient. +--- + fetchmail.h | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/fetchmail.h b/fetchmail.h +index bebb094d..6ff23919 100644 +--- a/fetchmail.h ++++ b/fetchmail.h +@@ -104,7 +104,7 @@ enum authenticators { + + #define NAMELEN 64 /* max username length */ + /* oauth2 access tokens seem to be about 130 characters; make this longer: */ +-#define PASSWORDLEN 256 /* max password length */ ++#define PASSWORDLEN 4096 /* max password length; oauth2 tokens have no maximum length */ + #define DIGESTLEN 33 /* length of MD5 digest */ + + /* exit code values */ + diff --git a/fetchmail-oauth2-c-calculate-and-pass-in-correct-buffer-size-to-to64frombits.patch b/fetchmail-oauth2-c-calculate-and-pass-in-correct-buffer-size-to-to64frombits.patch new file mode 100644 index 0000000..e681c89 --- /dev/null +++ b/fetchmail-oauth2-c-calculate-and-pass-in-correct-buffer-size-to-to64frombits.patch @@ -0,0 +1,38 @@ +From: Matthew Ogilvie +Date: Fri, 21 Dec 2018 09:01:40 -0700 +Subject: oauth2.c: calculate and pass in correct buffer size to to64frombits() +Git-repo: https://gitlab.com/fetchmail/fetchmail.git +Git-commit: 914ee333c73baa3c58d1e819ff4d66052e663335 + +Also allocate the actual needed size instead of an excessively large +approximate size. + +--- + oauth2.c | 6 ++++-- + 1 file changed, 4 insertions(+), 2 deletions(-) + +diff --git a/oauth2.c b/oauth2.c +index 4f7a072a..addc91aa 100644 +--- a/oauth2.c ++++ b/oauth2.c +@@ -34,6 +34,7 @@ char *get_oauth2_string(struct query *ctl,flag xoauth2) + int oauth2len; + + char *oauth2b64; ++ size_t oauth2b64alloc; + + oauth2len = strlen(ctl->remotename) + strlen(ctl->password) + 32; + oauth2str = (char *)xmalloc(oauth2len); +@@ -52,8 +53,9 @@ char *get_oauth2_string(struct query *ctl,flag xoauth2) + ctl->password); + } + +- oauth2b64 = (char *)xmalloc(2*strlen(oauth2str)+8); +- to64frombits(oauth2b64, oauth2str, strlen(oauth2str), oauth2len); ++ oauth2b64alloc = query_to64_outsize(strlen(oauth2str)); ++ oauth2b64 = (char *)xmalloc(oauth2b64alloc); ++ to64frombits(oauth2b64, oauth2str, strlen(oauth2str), oauth2b64alloc); + + memset(oauth2str, 0x55, strlen(oauth2str)); + free(oauth2str); + diff --git a/fetchmail-re-read-passwordfile-on-every-poll.patch b/fetchmail-re-read-passwordfile-on-every-poll.patch new file mode 100644 index 0000000..680f0a9 --- /dev/null +++ b/fetchmail-re-read-passwordfile-on-every-poll.patch @@ -0,0 +1,175 @@ +From: Matthew Ogilvie +Date: Fri, 9 Jun 2017 18:20:40 -0600 +Subject: re-read passwordfile on every poll +Git-repo: https://gitlab.com/fetchmail/fetchmail.git +Git-commit: c2b96715bb39b9cfd1c751eae6b0111bed9c8581 + +--- + fetchmail.c | 101 +++++++++++++++++++++++++++++++++----------------- + fetchmail.man | 9 ++--- + 2 files changed, 70 insertions(+), 40 deletions(-) + +diff --git a/fetchmail.c b/fetchmail.c +index e2828a4f..eb277c28 100644 +--- a/fetchmail.c ++++ b/fetchmail.c +@@ -586,48 +586,19 @@ int main(int argc, char **argv) + } + + ctl->password = xstrdup(msg); ++ ctl->passwordfile = NULL; + memset(msg, 0x55, mi-msg); + } else if (ctl->passwordfile) { +- int fd = open(ctl->passwordfile, O_RDONLY); +- char msg[PASSWORDLEN+1]; +- char *newline; +- int res; +- +- if (fd == -1) { +- int saveErrno = errno; +- fprintf(stderr, +- GT_("fetchmail: unable to open %s: %s\n"), +- ctl->passwordfile, +- strerror(saveErrno)); +- return PS_AUTHFAIL; +- } +- +- res = read(fd, msg, sizeof(msg)-1); +- if (res == -1 || close(fd) == -1) { ++ if (access(ctl->passwordfile, R_OK) != 0) { + int saveErrno = errno; + fprintf(stderr, +- GT_("fetchmail: error reading %s: %s\n"), ++ GT_("fetchmail: unable to access %s: %s\n"), + ctl->passwordfile, + strerror(saveErrno)); + return PS_AUTHFAIL; + } +- msg[res] = '\0'; +- +- newline = memchr(msg, '\n', res); +- if (newline != NULL) { +- *newline = '\0'; +- } +- +- if (strlen(msg) == 0) { +- fprintf(stderr, +- GT_("fetchmail: empty password read from %s\n"), +- ctl->passwordfile); +- memset(msg, 0x55, res); +- return PS_AUTHFAIL; +- } +- +- ctl->password = xstrdup(msg); +- memset(msg, 0x55, res); ++ ctl->password = xstrdup("dummy"); ++ /* file will be read/re-read on each poll interval below */ + } else if (!isatty(0)) { + fprintf(stderr, + GT_("fetchmail: can't find a password for %s@%s.\n"), +@@ -643,6 +614,8 @@ int main(int argc, char **argv) + ctl->password = xstrdup((char *)fm_getpassword(tmpbuf)); + free(tmpbuf); + } ++ } else { ++ ctl->passwordfile = NULL; + } + } + +@@ -843,6 +816,65 @@ int main(int argc, char **argv) + + dofastuidl = 0; /* this is reset in the driver if required */ + ++ if (ctl->passwordfile) { ++ int fd = open(ctl->passwordfile, O_RDONLY); ++ char msg[PASSWORDLEN+1]; ++ char *newline; ++ int res; ++ ++ if (fd == -1) { ++ int saveErrno = errno; ++ report(stderr, ++ GT_("fetchmail: unable to open %s: %s\n"), ++ ctl->passwordfile, ++ strerror(saveErrno)); ++ continue; ++ } ++ ++ res = read(fd, msg, sizeof(msg)-1); ++ close(fd); ++ if (res == -1) { ++ int saveErrno = errno; ++ report(stderr, ++ GT_("fetchmail: error reading %s: %s\n"), ++ ctl->passwordfile, ++ strerror(saveErrno)); ++ continue; ++ } ++ msg[res] = '\0'; ++ ++ newline = memchr(msg, '\n', res); ++ if (newline != NULL) { ++ *newline = '\0'; ++ } ++ ++ if (strlen(msg) == 0) { ++ report(stderr, ++ GT_("fetchmail: empty password read from %s\n"), ++ ctl->passwordfile); ++ memset(msg, 0x55, res); ++ continue; ++ } ++ ++ if (ctl->password) { ++ memset(ctl->password, 0x55, strlen(ctl->password)); ++ xfree(ctl->password); ++ } ++ ctl->password = xstrdup(msg); ++ memset(msg, 0x55, res); ++ } ++ ++ if (!ctl->password) { ++ /* This shouldn't be reachable (all cases caught ++ * earlier), but keep it for safety since there ++ * are many cases. ++ */ ++ report(stderr, ++ GT_("password is unexpectedly NULL querying %s\n"), ++ ctl->server.pollname); ++ continue; ++ } ++ + querystatus = query_host(ctl); + + if (NUM_NONZERO(ctl->fastuidl)) +diff --git a/fetchmail.man b/fetchmail.man +index b6309d40..6b375d89 100644 +--- a/fetchmail.man ++++ b/fetchmail.man +@@ -877,12 +877,9 @@ See USER AUTHENTICATION below for a complete description. + .br + Specifies a file name from which to read the first line to use as the password. + Useful if something changes the password/token often without regenerating a +-long fetchmailrc file, such as with typical xoauth2 authentication tokens. ++long fetchmailrc file, such as with typical oauth2 authentication tokens. + Protect the file with appropriate permissions to avoid leaking your password. +-Fetchmail might not re-read the file in daemon mode (-d) unless the +-fetchmailrc file also changes, so it might make sense to run it in +-non-daemon mode from some other background process (cron and/or whatever +-updates the password). ++Fetchmail will re-read the file for each poll when in daemon mode. + .TP + .B \-\-passwordfd + (Keyword: passwordfd) +@@ -895,7 +892,7 @@ although it could also be a redirected input file + (equivalent to "fetchmail \-\-passwordfd 5 5 +Date: Fri, 30 Jun 2017 02:35:12 -0600 +Subject: support oauthbearer/xoauth2 with pop3 +Git-repo: https://gitlab.com/fetchmail/fetchmail.git +Git-commit: 7b5c56f0fa3acb4c5589a4747c1921a311d8a464 + +(Also factor out some common imap/pop3 oauth2 code.) +--- + Makefile.am | 2 +- + fetchmail.man | 5 +-- + imap.c | 53 +++------------------- + oauth2.c | 61 +++++++++++++++++++++++++ + oauth2.h | 6 +++ + pop3.c | 122 ++++++++++++++++++++++++++++++++++++++++++++++++-- + 6 files changed, 196 insertions(+), 53 deletions(-) + create mode 100644 oauth2.c + create mode 100644 oauth2.h + +diff --git a/Makefile.am b/Makefile.am +index 1e800085..d747f895 100644 +--- a/Makefile.am ++++ b/Makefile.am +@@ -54,7 +54,7 @@ fetchmail_SOURCES= fetchmail.h getopt.h \ + fetchmail.c env.c idle.c options.c daemon.c \ + driver.c transact.c sink.c smtp.c \ + idlist.c uid.c mxget.c md5ify.c cram.c gssapi.c \ +- opie.c interface.c netrc.c \ ++ oauth2.c opie.c interface.c netrc.c \ + unmime.c conf.c checkalias.c uid_db.h uid_db.c\ + lock.h lock.c \ + rcfile_l.l rcfile_y.y \ +diff --git a/fetchmail.man b/fetchmail.man +index d128ece1..aece716e 100644 +--- a/fetchmail.man ++++ b/fetchmail.man +@@ -928,7 +928,7 @@ This option permits you to specify an authentication type (see USER + \&\fBpassword\fP, \fBkerberos_v5\fP, \fBkerberos\fP (or, for + excruciating exactness, \fBkerberos_v4\fP), \fBgssapi\fP, + \fBcram\-md5\fP, \fBotp\fP, \fBntlm\fP, \fBmsn\fP (only for POP3), +-\fBexternal\fP (only IMAP), \fBssh\fP and \fBoauthbearer\fP (only IMAP). ++\fBexternal\fP (only IMAP), \fBssh\fP and \fBoauthbearer\fP (requires token). + When \fBany\fP (the default) is specified, fetchmail tries + first methods that don't require a password (EXTERNAL, GSSAPI, KERBEROS\ IV, + KERBEROS\ 5); then it looks for methods that mask your password +@@ -2222,8 +2222,7 @@ Legal protocol identifiers for use with the 'protocol' keyword are: + Legal authentication types are 'any', 'password', 'kerberos', + \&'kerberos_v4', 'kerberos_v5' and 'gssapi', 'cram\-md5', 'otp', 'msn' + (only for POP3), 'ntlm', 'ssh', 'external' (only IMAP), +-'oauthbearer' (only for IMAP; requires authentication token in +-place of password). ++'oauthbearer' (requires authentication token in place of password). + The 'password' type specifies + authentication by normal transmission of a password (the password may be + plain text or subject to protocol-specific encryption as in CRAM-MD5); +diff --git a/imap.c b/imap.c +index 0ab10d31..e38706f5 100644 +--- a/imap.c ++++ b/imap.c +@@ -14,6 +14,7 @@ + #include + #include + #endif ++#include "oauth2.h" + #include "socket.h" + + #include "i18n.h" +@@ -329,63 +330,23 @@ static int do_imap_ntlm(int sock, struct query *ctl) + + static int do_imap_oauthbearer(int sock, struct query *ctl,flag xoauth2) + { +- /* Implements relevant parts of RFC-7628, RFC-6750, and +- * https://developers.google.com/gmail/imap/xoauth2-protocol +- * +- * This assumes something external manages obtaining an up-to-date +- * authentication/bearer token and arranging for it to be in +- * ctl->password. This may involve renewing it ahead of time if +- * necessary using a renewal token that fetchmail knows nothing about. +- * See: +- * https://github.com/google/gmail-oauth2-tools/wiki/OAuth2DotPyRunThrough +- */ +- const char *name; +- char *oauth2str; +- int oauth2len; +- int saved_suppress_tags = suppress_tags; +- +- char *oauth2b64; +- ++ char *oauth2str = get_oauth2_string(ctl, xoauth2); ++ const char *name = xoauth2 ? "XOAUTH2" : "OAUTHBEARER"; + int ok; + +- oauth2len = strlen(ctl->remotename) + strlen(ctl->password) + 32; +- oauth2str = (char *)xmalloc(oauth2len); +- if (xoauth2) +- { +- snprintf(oauth2str, oauth2len, +- "user=%s\1auth=Bearer %s\1\1", +- ctl->remotename, +- ctl->password); +- name = "XOAUTH2"; +- } +- else +- { +- snprintf(oauth2str, oauth2len, +- "n,a=%s,\1auth=Bearer %s\1\1", +- ctl->remotename, +- ctl->password); +- name = "OAUTHBEARER"; +- } +- +- oauth2b64 = (char *)xmalloc(2*strlen(oauth2str)+8); +- to64frombits(oauth2b64, oauth2str, strlen(oauth2str)); +- +- memset(oauth2str, 0x55, strlen(oauth2str)); +- free(oauth2str); +- + /* Protect the access token like a password in logs, despite the + * usually-short expiration time and base64 encoding: + */ +- strlcpy(shroud, oauth2b64, sizeof(shroud)); ++ strlcpy(shroud, oauth2str, sizeof(shroud)); + + plus_cont_context = IPLUS_OAUTHBEARER; +- ok = gen_transact(sock, "AUTHENTICATE %s %s", name, oauth2b64); ++ ok = gen_transact(sock, "AUTHENTICATE %s %s", name, oauth2str); + plus_cont_context = IPLUS_NONE; + + memset(shroud, 0x55, sizeof(shroud)); + shroud[0] = '\0'; +- memset(oauth2b64, 0x55, strlen(oauth2b64)); +- free(oauth2b64); ++ memset(oauth2str, 0x55, strlen(oauth2str)); ++ free(oauth2str); + + return ok; + } +diff --git a/oauth2.c b/oauth2.c +new file mode 100644 +index 00000000..a8a324b8 +--- /dev/null ++++ b/oauth2.c +@@ -0,0 +1,61 @@ ++/* ++ * oauth2.c -- oauthbearer and xoauth2 support ++ * ++ * Copyright 2017 by Matthew Ogilvie ++ * For license terms, see the file COPYING in this directory. ++ */ ++ ++#include "config.h" ++#include "fetchmail.h" ++#include "oauth2.h" ++ ++#include ++#include ++ ++char *get_oauth2_string(struct query *ctl,flag xoauth2) ++{ ++ /* Implements the bearer token string based for a ++ * combination of RFC-7628 (ouath sasl, with ++ * examples for imap only), RFC-6750 (oauth2), and ++ * RFC-5034 (pop sasl), as implemented by gmail and others. ++ * ++ * Also supports xoauth2, which is just a couple of minor variariations. ++ * https://developers.google.com/gmail/imap/xoauth2-protocol ++ * ++ * This assumes something external manages obtaining an up-to-date ++ * authentication/bearer token and arranging for it to be in ++ * ctl->password. This may involve renewing it ahead of time if ++ * necessary using a renewal token that fetchmail knows nothing about. ++ * See: ++ * https://github.com/google/gmail-oauth2-tools/wiki/OAuth2DotPyRunThrough ++ */ ++ char *oauth2str; ++ int oauth2len; ++ ++ char *oauth2b64; ++ ++ oauth2len = strlen(ctl->remotename) + strlen(ctl->password) + 32; ++ oauth2str = (char *)xmalloc(oauth2len); ++ if (xoauth2) ++ { ++ snprintf(oauth2str, oauth2len, ++ "user=%s\1auth=Bearer %s\1\1", ++ ctl->remotename, ++ ctl->password); ++ } ++ else ++ { ++ snprintf(oauth2str, oauth2len, ++ "n,a=%s,\1auth=Bearer %s\1\1", ++ ctl->remotename, ++ ctl->password); ++ } ++ ++ oauth2b64 = (char *)xmalloc(2*strlen(oauth2str)+8); ++ to64frombits(oauth2b64, oauth2str, strlen(oauth2str)); ++ ++ memset(oauth2str, 0x55, strlen(oauth2str)); ++ free(oauth2str); ++ ++ return oauth2b64; ++} +diff --git a/oauth2.h b/oauth2.h +new file mode 100644 +index 00000000..67ebfd6e +--- /dev/null ++++ b/oauth2.h +@@ -0,0 +1,6 @@ ++#ifndef OAUTH2_H ++#define OAUTH2_H ++ ++char *get_oauth2_string(struct query *ctl,flag xoauth2); ++ ++#endif /*OAUTH2_H*/ +diff --git a/pop3.c b/pop3.c +index 076d890e..06fc0a0d 100644 +--- a/pop3.c ++++ b/pop3.c +@@ -15,6 +15,7 @@ + #include + + #include "fetchmail.h" ++#include "oauth2.h" + #include "socket.h" + #include "i18n.h" + #include "uid_db.h" +@@ -55,6 +56,10 @@ flag has_ntlm = FALSE; + #ifdef SSL_ENABLE + static flag has_stls = FALSE; + #endif /* SSL_ENABLE */ ++static flag has_oauthbearer = FALSE; ++static flag has_xoauth2 = FALSE; ++ ++static const char *next_sasl_resp = NULL; + + /* mailbox variables initialized in pop3_getrange() */ + static int last; +@@ -110,12 +115,65 @@ static int pop3_ok (int sock, char *argbuf) + char buf [POPBUFSIZE+1]; + char *bufp; + +- if ((ok = gen_recv(sock, buf, sizeof(buf))) == 0) ++ while ((ok = gen_recv(sock, buf, sizeof(buf))) == 0) + { bufp = buf; +- if (*bufp == '+' || *bufp == '-') ++ if (*bufp == '+') ++ { + bufp++; ++ if (*bufp == ' ' && next_sasl_resp != NULL) ++ { ++ /* Currently only used for OAUTHBEARER/XOAUTH2, and only ++ * rarely even then. ++ * ++ * This is the only case where the top while() actually ++ * loops. ++ * ++ * For OAUTHBEARER, data aftetr '+ ' is probably ++ * base64-encoded JSON with some HTTP-related error details. ++ */ ++ if (*next_sasl_resp != '\0') ++ SockWrite(sock, next_sasl_resp, strlen(next_sasl_resp)); ++ SockWrite(sock, "\r\n", 2); ++ if (outlevel >= O_MONITOR) ++ { ++ const char *found; ++ if (shroud[0] && (found = strstr(next_sasl_resp, shroud))) ++ { ++ /* enshroud() without copies, and avoid ++ * confusing with a genuine "*" (cancel). ++ */ ++ report(stdout, "POP3> %.*s[SHROUDED]%s\n", ++ (int)(found-next_sasl_resp), next_sasl_resp, ++ found+strlen(shroud)); ++ } ++ else ++ { ++ report(stdout, "POP3> %s\n", next_sasl_resp); ++ } ++ } ++ ++ if (*next_sasl_resp == '\0' || *next_sasl_resp == '*') ++ { ++ /* No more responses expected, cancel AUTH command if ++ * more responses requested. ++ */ ++ next_sasl_resp = "*"; ++ } ++ else ++ { ++ next_sasl_resp = ""; ++ } ++ continue; ++ } ++ } ++ else if (*bufp == '-') ++ { ++ bufp++; ++ } + else ++ { + return(PS_PROTOCOL); ++ } + + while (isalpha((unsigned char)*bufp)) + bufp++; +@@ -184,6 +242,8 @@ static int pop3_ok (int sock, char *argbuf) + #endif + if (argbuf != NULL) + strcpy(argbuf,bufp); ++ ++ break; + } + + return(ok); +@@ -212,11 +272,13 @@ static int capa_probe(int sock) + #ifdef NTLM_ENABLE + has_ntlm = FALSE; + #endif /* NTLM_ENABLE */ ++ has_oauthbearer = FALSE; ++ has_xoauth2 = FALSE; + + ok = gen_transact(sock, "CAPA"); + if (ok == PS_SUCCESS) + { +- char buffer[64]; ++ char buffer[128]; + + /* determine what authentication methods we have available */ + while ((ok = gen_recv(sock, buffer, sizeof(buffer))) == 0) +@@ -246,6 +308,12 @@ static int capa_probe(int sock) + + if (strstr(buffer, "CRAM-MD5")) + has_cram = TRUE; ++ ++ if (strstr(buffer, "OAUTHBEARER")) ++ has_oauthbearer = TRUE; ++ ++ if (strstr(buffer, "XOAUTH2")) ++ has_xoauth2 = TRUE; + } + } + done_capa = TRUE; +@@ -312,6 +380,40 @@ static int do_apop(int sock, struct query *ctl, char *greeting) + peek_capable = !ctl->fetchall && (!ctl->keep || ctl->server.uidl); + } + ++static int do_oauthbearer(int sock, struct query *ctl, flag xoauth2) ++{ ++ char *oauth2str = get_oauth2_string(ctl, xoauth2); ++ const char *name = xoauth2 ? "XOAUTH2" : "OAUTHBEARER"; ++ int ok; ++ ++ /* Protect the access token like a password in logs, despite the ++ * usually-short expiration time and base64 encoding: ++ */ ++ strlcpy(shroud, oauth2str, sizeof(shroud)); ++ ++ if (4+1+1+2+strlen(name)+strlen(oauth2str) <= 255) ++ { ++ next_sasl_resp = ""; ++ ok = gen_transact(sock, "AUTH %s %s", name, oauth2str); ++ } ++ else ++ { ++ /* Too long to use "initial client response" (RFC-5034 section 4, ++ * referencing RFC-4422 section 4). ++ */ ++ next_sasl_resp = oauth2str; ++ ok = gen_transact(sock, "AUTH %s", name); ++ } ++ next_sasl_resp = NULL; ++ ++ memset(shroud, 0x55, sizeof(shroud)); ++ shroud[0] = '\0'; ++ memset(oauth2str, 0x55, strlen(oauth2str)); ++ free(oauth2str); ++ ++ return ok; ++} ++ + static int pop3_getauth(int sock, struct query *ctl, char *greeting) + /* apply for connection authorization */ + { +@@ -436,6 +538,7 @@ static int pop3_getauth(int sock, struct query *ctl, char *greeting) + (ctl->server.authenticate == A_KERBEROS_V5) || + (ctl->server.authenticate == A_OTP) || + (ctl->server.authenticate == A_CRAM_MD5) || ++ (ctl->server.authenticate == A_OAUTHBEARER) || + maybe_starttls(ctl)) + { + if ((ok = capa_probe(sock)) != PS_SUCCESS) +@@ -540,6 +643,19 @@ static int pop3_getauth(int sock, struct query *ctl, char *greeting) + /* + * OK, we have an authentication type now. + */ ++ if (ctl->server.authenticate == A_OAUTHBEARER) ++ { ++ if (has_oauthbearer || !has_xoauth2) ++ { ++ ok = do_oauthbearer(sock, ctl, FALSE); /* OAUTHBEARER */ ++ } ++ if (ok != PS_SUCCESS && has_xoauth2) ++ { ++ ok = do_oauthbearer(sock, ctl, TRUE); /* XOAUTH2 */ ++ } ++ break; ++ } ++ + #if defined(KERBEROS_V4) + /* + * Servers doing KPOP have to go through a dummy login sequence +-- +2.31.1 + diff --git a/fetchmail.changes b/fetchmail.changes index 209e68c..c9939e6 100644 --- a/fetchmail.changes +++ b/fetchmail.changes @@ -1,3 +1,35 @@ +------------------------------------------------------------------- +Thu May 13 16:57:09 UTC 2021 - Jeff Mahoney + +- Backported support for OAUTH2 authentication from Fetchmail 7.0. + - add imap oauthbearer support + - support oauthbearer/xoauth2 with pop3 + - add passwordfile and passwordfd options + - add contrib/fetchnmail-oauth2.py token acquisition utility + - FAQ: list gmail options including oauthbearer and app password + - give each ctl it's own copy of password + - re-read passwordfile on every poll + - add query_to64_outsize() utility function + - Chase and integrate interface change. + - oauth2.c: calculate and pass in correct buffer size to to64frombits() + - Increase max password length to handle oauth tokens + - Bump max. passwordlen to 10000 bytes. + - Add README.OAUTH2 +- Added patches: + * fetchmail-add-imap-oauthbearer-support.patch + * fetchmail-support-oauthbearer-xoauth2-with-pop3.patch + * fetchmail-add-passwordfile-and-passwordfd-options.patch + * fetchmail-add-contrib-fetchnmail-oauth2.py-token-acquisition-u.patch + * fetchmail-FAQ-list-gmail-options-including-oauthbearer-and-app.patch + * fetchmail-give-each-ctl-it-s-own-copy-of-password.patch + * fetchmail-re-read-passwordfile-on-every-poll.patch + * fetchmail-add-query_to64_outsize-utility-function.patch + * fetchmail-chase-and-integrate-interface-change.patch + * fetchmail-oauth2-c-calculate-and-pass-in-correct-buffer-size-to-to64frombits.patch + * fetchmail-increase-max-password-length-to-handle-oauth-tokens.patch + * fetchmail-bump-max-passwordlen-to-1bytes.patch + * fetchmail-add-readme-oauth2-issue-27.patch + ------------------------------------------------------------------- Sat May 8 19:55:09 UTC 2021 - Dirk Müller diff --git a/fetchmail.spec b/fetchmail.spec index db304c5..3a0914d 100644 --- a/fetchmail.spec +++ b/fetchmail.spec @@ -26,7 +26,8 @@ Release: 0 Summary: Full-Featured POP and IMAP Mail Retrieval Daemon License: GPL-2.0-or-later URL: https://www.fetchmail.info/ -Source: https://sourceforge.net/projects/%{name}/files/branch_6.4/%{name}-%{version}.tar.xz +#Source: fetchmail-7.0.0-alpha8.tar.xz +Source0: https://sourceforge.net/projects/%{name}/files/branch_6.4/%{name}-%{version}.tar.xz Source1: https://sourceforge.net/projects/%{name}/files/branch_6.4/%{name}-%{version}.tar.xz.asc Source2: %{name}.logrotate Source3: sysconfig.%{name} @@ -35,8 +36,24 @@ Source6: %{name}.service Source7: %{name}.tmpfiles Source8: %{name}.exec Source9: %{name}.sysusers + Patch0: fetchmail-6.3.8-smtp_errors.patch +Patch1: fetchmail-add-imap-oauthbearer-support.patch +Patch2: fetchmail-support-oauthbearer-xoauth2-with-pop3.patch +Patch3: fetchmail-add-passwordfile-and-passwordfd-options.patch +Patch4: fetchmail-add-contrib-fetchnmail-oauth2.py-token-acquisition-u.patch +Patch5: fetchmail-FAQ-list-gmail-options-including-oauthbearer-and-app.patch +Patch6: fetchmail-give-each-ctl-it-s-own-copy-of-password.patch +Patch7: fetchmail-re-read-passwordfile-on-every-poll.patch +Patch8: fetchmail-add-query_to64_outsize-utility-function.patch +Patch9: fetchmail-chase-and-integrate-interface-change.patch +Patch10: fetchmail-oauth2-c-calculate-and-pass-in-correct-buffer-size-to-to64frombits.patch +Patch11: fetchmail-increase-max-password-length-to-handle-oauth-tokens.patch +Patch12: fetchmail-bump-max-passwordlen-to-1bytes.patch +Patch13: fetchmail-add-readme-oauth2-issue-27.patch BuildRequires: automake +BuildRequires: bison +BuildRequires: flex BuildRequires: krb5-devel BuildRequires: openssl-devel BuildRequires: opie @@ -78,6 +95,19 @@ files (.fetchmailrc). %prep %setup -q %patch0 -p1 +%patch1 -p1 +%patch2 -p1 +%patch3 -p1 +%patch4 -p1 +%patch5 -p1 +%patch6 -p1 +%patch7 -p1 +%patch8 -p1 +%patch9 -p1 +%patch10 -p1 +%patch11 -p1 +%patch12 -p1 +%patch13 -p1 cp -a %{SOURCE2} %{SOURCE3} . ACLOCAL="aclocal -I m4 -I m4-local" autoreconf -fvi @@ -143,7 +173,7 @@ rm -r contrib/gai* %files -f %{name}.lang %license COPYING -%doc FAQ FEATURES NEWS NOTES OLDNEWS README README.NTLM README.SSL README.SSL-SERVER TODO contrib *.html *.txt *.pdf +%doc FAQ FEATURES NEWS NOTES OLDNEWS README README.NTLM README.OAUTH2 README.SSL README.SSL-SERVER TODO contrib *.html *.txt *.pdf %{_bindir}/fetchmail %dir %attr(0700, fetchmail, fetchmail) %{_localstatedir}/lib/fetchmail %ghost %attr(0600, fetchmail, root) %{_localstatedir}/log/fetchmail