[NEW:2020-07-18] ERPNext v12 manual install on ubuntu 16/18/20 with Python v3.x

#This part is OPTIONAL!
sshPort="22244";
ipADDR="192.168.1.244";
sed -i '/#Port 22/Port $sshPort/' /etc/ssh/sshd_config;
systemctl restart ssh

ssh user001@$ipADDR -p $sshPort; #if you didn't change the listener ports for SSH, then you don't need the "-p xxxxx" part

#Switch to Root User
sudo -i;
apt update -y && apt upgrade -y;

#Set some variables to make life a bit easier - NB!!! these values are NOT safe for live systems, so please DON'T USE THEM!!!!!
RootPWD="rootpassword";
ERPNextPWD="erpnextpassword";
MySQLpwd="mysqlpassword";       #This will be used in the mysql_secure_installation command a few lines down
adminPWD="adminpassword";
#echo "root:$RootPWD;erpnext:$ERPNextPWD;mysql:$MySQLpwd;"  #uncomment this line to show these#4DBG#

#Install essential utilities
sudo apt install -y sshfs gcc wget binutils net-tools dnsutils lsof mc \
    curl libssl-dev libffi-dev network-manager \
    git-core build-essential software-properties-common \
    dirmngr apt-transport-https ca-certificates \
    ;

#Install mariadb/mysql
sudo apt-key adv --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24C74CD1D8 #Signing key for mariaDB
sudo apt install mariadb-client mariadb-server -y
#1) Test the install: 
   mysql -u root
   exit
#2) Also good idea to do the GRANT items here after 

mysql_secure_installation   #You'll use the password you set here in the lines below
MySQLusr="root";            #as set in the above command [mysql_secure_installation]
MySQLpwd="mysqlpassword";   #as set in the above command [mysql_secure_installation]
#echo $MySQLusr $MySQLpwd #4DBG#
commandToRun="GRANT ALL PRIVILEGES ON *.* TO \"$MySQLusr\"@\"localhost\" IDENTIFIED BY \"$MySQLpwd\" WITH GRANT OPTION";      #echo $commandToRun;  #4DBG#
  mysql -u$MySQLusr -p$MySQLpwd -Bse "$commandToRun";
commandToRun="GRANT ALL PRIVILEGES ON *.* TO \"$MySQLusr\"@\"127.%.%.%\" IDENTIFIED BY \"$MySQLpwd\" WITH GRANT OPTION";      #echo $commandToRun;  #4DBG#
  mysql -u$MySQLusr -p$MySQLpwd -Bse "$commandToRun";
#the command below needs to be tuned to YOUR LAN/subnet - in my case, machines on the localnet can access the DB - but this is OPTIONAL
commandToRun="GRANT ALL PRIVILEGES ON *.* TO \"$MySQLusr\"@\"192.168.%.%\" IDENTIFIED BY \"$MySQLpwd\" WITH GRANT OPTION";    #echo $commandToRun; #4DBG#
  mysql -u$MySQLusr -p$MySQLpwd -Bse "$commandToRun";


#Tune the DB (should be done before the frappe install, otherwise it will warn you to make these changes before it proceeds)
#I had some issues because I used .conf instead of .cnf, so check your system's preference/requirement
cat /etc/mysql/my.cnf | awk '/conf.d/ {print $3}'   
#You can use `nano /etc/mysql/my.cnf` - OR - create a conf.d version in one of /etc/mysql/conf.d/ or /etc/mysql/mariadb.conf.d/
    #https://mariadb.com/kb/en/configuring-mariadb-with-option-files/
    #   All option file names MUST end in  .cnf  on Unix-like operating systems.
    #NB! if you're running mariadb and you specify the directives in /etc/mysql/conf.d/*.cnf, some of them get ignored (eg. collation settings)
  # - I don't know why, but eg.collation_server gets ignored if set via /etc/mysql/conf.d/*.cnf but works perfectly via /etc/mysql/mariadb.conf.d/*.cnf
  # - I have had no issues on ubuntu 18.04,20.04 using /etc/mysql/mariadb.conf.d/mariadb4ERPNext.cnf
  #     BUT on ubuntu 16.04 I have to use /etc/mysql/my.cnf and NOT /etc/mysql/mariadb.conf.d/*.cnf|/etc/mysql/conf.d/*.cnf

#A few parameters and file locations are conditional to the OS release number, so let's get that now
ubuntuRELEASE=$(lsb_release --release --short | cut -d. -f1); printf "$ubuntuRELEASE\n"
# cat /etc/lsb-release | grep "RELEASE" | cut -d= -f2;          # cat /etc/os-release | grep "VERSION_ID" | cut -d= -f2 | sed s/\"//g;
#If the first one doesn't work, you could try these alternatives

#The figures should be adjusted to suit YOUR system - which is guesstimated and implemented here...
physRAM=$(vmstat -s --unit M | awk '/total memory/ { print int( ($1)/1000 + 0.5) * 1024 } '); #printf "RAM:$physRAM\n";#4DBG#
innodbBufferPoolSize=$(($physRAM/2));             innodbBufferPoolSizeM=$(echo "$innodbBufferPoolSize""M");
innodbLogFileSize=$(($innodbBufferPoolSize/4));   innodbLogFileSizeM=$(echo "$innodbLogFileSize""M");
innodbLogBufferSize=$(($innodbLogFileSize/2));    innodbLogBufferSizeM=$(echo "$innodbLogBufferSize""M");
printf "$innodbBufferPoolSizeM\t$innodbLogFileSizeM\t$innodbLogBufferSizeM\n";#4DBG#

#First create a temp file (in overwrite mode) with the required content
subDest="/etc/mysql/mariadb.conf.d/"; [[ -d $subDest ]] && echo "$subDest exists" || mkdir -p $subDest;
mysqlCONF_tmp="/etc/mysql/mariadb.conf.d/mariadb4ERPNext.cnf";  #printf "$mysqlCONF_tmp\n";#4DBG#
cat <<EOF > $mysqlCONF_tmp
[mysqld]
innodb_file_format=barracuda
innodb_file_per_table=1
innodb_large_prefix=1
character_set_client_handshake = FALSE
character_set_server = utf8mb4
collation_server = utf8mb4_unicode_ci
# Set buffer pool size to 50_80% of your computer's memory
innodb_buffer_pool_size=$innodbBufferPoolSizeM
# Set the log file size to about 25% of the buffer pool size
innodb_log_file_size=$innodbLogFileSizeM
innodb_log_buffer_size=$innodbLogBufferSizeM
# the default here is 4MB, or maybe ??16MB?? in more recent versions, which sometimes allows for disconnects in the middle of transactions
max_allowed_packet=32M

[mysql]
default_character_set = utf8mb4
EOF
# cat $mysqlCONF_tmp;#4DBG#

#put the config_tmp file into the right place depending on the OS-release (v16 wants /etc/mysql/my.cnf; v18,v20 can use *conf.d/*.cnf)
if [[ $ubuntuRELEASE -ge 18 ]]; then
  mysqlCONF="/etc/mysql/mariadb.conf.d/mariadb4ERPNext.cnf";  #sameFile mode
else
  mysqlCONF="/etc/mysql/my.cnf";  cat $mysqlCONF_tmp >> $mysqlCONF; #append mode
fi;
#printf "$mysqlCONF\n";#4DBG#
#cat "$mysqlCONF";#4DBG#
sudo chmod 644 $mysqlCONF;    #set permissions - according to mysql documentation, the DB will ignore cnf files with world writeable permissions

#On ubuntu 16, the mysqld service doesn't always stop when using the usual `systemctl stop mysql` and/or `service mysqld stop`, so this workaround is needed
if [[ $ubuntuRELEASE -eq 16 ]]; then
  /etc/init.d/mysql stop;       #stop the mysql/mariadb (generally doesn't seem to work on u16)
  SERVICE="mysqld_safe";  mysqlsafePID=$(pgrep -x "$SERVICE"); printf "$mysqlPID\n";#4DBG#
  SERVICE="mysqld";       mysqlPID=$(pgrep -x "$SERVICE"); printf "$mysqlPID\n";#4DBG#
  SERVICE="logger";       mylogPID=$(pgrep -x "$SERVICE"); printf "$mylogPID\n";#4DBG#
  sudo kill -9 "$mysqlsafePID" "$mysqlPID" "$mylogPID"; #kill BOTH processes, otherwise it fails to stop
  sleep 3; #short pause, just to be sure
  sudo systemctl restart mysql;   #Restart the mysql/mariadb 
else
  sudo systemctl restart mysql;   #Restart the mysql/mariadb 
fi;
#AND# check that the values were accepted/loaded
commandToRun="SHOW VARIABLES;";      #echo $commandToRun;  #4DBG#
mysql -u$MySQLusr -p$MySQLpwd -Bse "$commandToRun" | grep "innodb_file_format\|innodb_buffer_pool_size\|innodb_log_\(file\|buffer\)_size\|innodb_large\|^max_allowed\|character_set_server\|collation_server";

[[ $ubuntuRELEASE -ge 18 ]] && mariaDBclient="libmariadbclient-dev" || mariaDBclient="libmariadb-client-lgpl-dev";  #printf "$mariaDBclient\n";#4DBG#
sudo apt install $mariaDBclient -y;

# Install web server/services  #apache2 shouldn't be running if this is a new install, but if it is, disable it, otherwise the nginx install will FAIL
SERVICE="apache2"; if pgrep -x "$SERVICE" >/dev/null; then sudo systemctl stop "$SERVICE"; fi;  
sudo apt install -y nginx;       #install nginx
myIP=$(ip -4 a | awk '/global/ {print $2}' | cut -d\/ -f1) && printf "Test the install by browsing to http://$myIP  (you should see the nginx default page)\n"
sudo apt install -y supervisor; #needed for production mode

# Install node and related utilities - I have stayed with node v12 (although v14 is available)
curl -sL https://deb.nodesource.com/setup_12.x | sudo bash -;   #####v12.x
sudo apt install nodejs -y;
# Install the Yarn package manager, run:
curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -;
echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list;
sudo apt update -y && sudo apt -y install yarn;
  node -v && npm -v && yarn -v;  #Test the install[optional]
# Redis
sudo apt install redis-server -y;
  redis-server -v;  #1- Test the install: 

# wkhtmltopdf        #apt install wkhtmltopdf -y #[NB! default install uses an "old" ~0.12.3 version which is not good]
sudo apt install -y libfontconfig fontconfig xvfb libxrender1 xfonts-base xfonts-75dpi; #required libraries
# wget https://github.com/wkhtmltopdf/wkhtmltopdf/releases/download/0.12.5/wkhtmltox_0.12.5-1.trusty_amd64.deb && sudo dpkg --install wkhtmltox_0.12.5-1.trusty_amd64.deb;
[[ $ubuntuRELEASE -eq 16 ]] && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.xenial_amd64.deb;
[[ $ubuntuRELEASE -eq 18 ]] && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.bionic_amd64.deb;
[[ $ubuntuRELEASE -eq 20 ]] && wget https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6-1/wkhtmltox_0.12.6-1.focal_amd64.deb;
[ -f $(ls ./wkhtmltox_0.12.*.deb) ] && sudo dpkg --install wkhtmltox_0.12.*_amd64.deb; 
wkhtmltopdf --version  #Test the install: wkhtmltopdf 0.12.6 (with patched qt)

ss -tnlp;   #check all the listeners/ports etc
            #You should have ssh/22, nginx/80, redis/6379, mysql/3306
#______________________________________________________________________________#
# Python related items - specifically, and carefully around the python3 side of things.
# In MOST cases below, python3->python will install the v2.7.x versions, but we're specifically avoiding that for this python3 preferred installation
sudo apt-get install -y virtualenv python3-dev python3-setuptools python3-pip \
    python3-mysqldb python3-cxx-dev python-pip-whl \
    python3-dateutil \
    ;
[[ $ubuntuRELEASE -eq 16 ]] && sudo apt-get install -y python3-pypdf2 || sudo apt-get install -y python3-pdfkit; #there isn't a python3-pdfkit(wkhtmltopdf/python wrapper) for u16, so I'm hoping this will work??
[[ $ubuntuRELEASE -eq 20 ]] && sudo apt-get install -y python3-testresources;
    #apt-cache search python | grep -i "pdf"
    #   python3-pypdf2 - Pure-Python library built as a PDF toolkit (Python 3)

###On debian Buster, I experienced a LOT of hassles with the install of MySQL-python
     #Have been struggling to get the pip install --upgrade mysql-python to work. 
pip3 install --upgrade mysqlclient   #this is OK, and/but can also be...
# Normally, to test, one would go into python, and then import MySQLdb, and try to connect. 
# The script works on u20 using "localhost", but fails on v16/v18 (works if you use "127.0.0.1")
# I create a .py file using the $vars set at the top and execute it, instead of the previous manual(ish) method. The blank line before main() is REQD!
cat <<EOF > ./connTest.py
import MySQLdb
def main():
  db = MySQLdb.connect(host = "127.0.0.1", user = "$MySQLusr", passwd = "$MySQLpwd", db = "mysql")
  if (db):
    print ("Connection successful")
  else:
    print ("Connection unsuccessful")

main()
EOF
# cat ./connTest.py  #4DBG#
python3 ./connTest.py
# rm -f ./connTest.py  #4DBG#Maybe only remove this if the connection succeeds
pip3 list | grep -i mysql;  #Check which pip/mysql modules are available...
  #  mysqlclient                   2.0.1
[[ $ubuntuRELEASE -eq 16 ]] && pip3 install --upgrade pip==19.3.1;  #to the best of my recollection, v19.3.1 works on u16+ERPNext12, but I may have to adjust this???#!ToCheck

erpnextuser="erpnext01";   #adjust to your preferred name!!!!!
sudo useradd -m -d /home/$erpnextuser -s /bin/bash $erpnextuser; passwd $erpnextuser;
#erpnextpassword
sudo adduser $erpnextuser sudo

#REBOOT server
reboot;

#Login as ##########ERPNext/FRAPPE########## user OR # su - erpnext01
erpnextuser="erpnext01";   #adjust to your preferred/created name from a few lines up!!!!!
sshPort="22"; ipADDR="192.168.1.244"; ssh $erpnextuser@$ipADDR -p $sshPort; #if you didn't change the listener ports for SSH, then you don't need the "-p xxxxx" part

#Set the variables again (since the reboot above would have killed them)
RootPWD="rootpassword";         #rootpassword
ERPNextPWD="erpnextpassword";   #erpnextpassword
MySQLpwd="mysqlpassword";       #mysqlpassword
adminPWD="adminpassword";       #adminpassword

cd ~
git clone https://github.com/frappe/bench bench-repo;
      #Normally, you'd only do it this way IF you're doing dev-mode
sudo pip3 install frappe-bench;     #NB!!! MUST be sudo'd!!! if you exclude the sudo part, then the sudo bench commands required later will fail!
#OR#
# sudo pip3 install -e bench-repo;    #NB!!! MUST be sudo'd!!! if you exclude the sudo part, then the sudo bench commands required later will fail!
      #from @revant_one on https://discuss.erpnext.com/
      #Using pip install -e ./bench-repo installs the bench from the locally cloned bench-repo directory.
      #That is called the editable mode. Because you can edit the bench source code and it’ll reflect when bench command is executed.
      #It is recommended if you’re developing bench itself or sending PR to bench repo.
      #If pip install frappe-bench is used, it’ll pull the pypi package and install it.
      #You’ll not be able to do bench development. Recommended when you know editing bench is not required. 
bench init --frappe-branch version-12 --python /usr/bin/python3 frappe-bench;

cd $HOME/frappe-bench;
erpnextSiteName="site1.local";
bench new-site $erpnextSiteName;
    # You'll be then prompted to input your MySQL root password.
      # IF this happens....
      #ERROR 1045 (28000): Access denied for user '_8de49ae55a02e7ac'@'localhost' (using password: YES)
      DBname=$(cat ~/frappe-bench/sites/$erpnextSiteName/site_config.json | awk '/db_name/ {print $2}' | sed "s/\"//g" | sed "s/,//g");
      # delete the DB & site, and then retry
      mysql -uroot -p$MySQLpwd -Bse "drop database $DBname;"
      rm -fR $HOME/frappe-bench/sites/$erpnextSiteName;
      bench new-site $erpnextSiteName;
    # You'll also be prompted to create a new password for the System Administrator.

# NB!!! ubuntu 20.04 # https://github.com/frappe/erpnext/issues/22424    # adityaduggal
if [[ $ubuntuRELEASE -eq 20 ]]; then
  cd $HOME/frappe-bench;
  ./env/bin/pip3 install numpy==1.18.5;
  ./env/bin/pip3 install pandas==0.24.2;
fi;
# Also note, this process takes quite long, so don't lose heart - just be patient - get yourself some coffee, and let it finish

bench get-app --branch version-12 erpnext;
    # bench get-app [--branch <branch>] <APP_REMOTE_URL|VALID_FRAPPE_APP>
./env/bin/pip3 install -e apps/erpnext/


# Installing Frappé Apps onto Sites
    # bench --site <MY_SITE> install-app <MY_APP_NAME>
bench --site $erpnextSiteName install-app erpnext;
#This happens most times... IF the bench start screen shows anything like
#Error: ENOSPC: System limit for number of file watchers reached, watch '/home/erpnext01/frappe-bench/apps/frappe/frappe/public/less/variables.less'
echo fs.inotify.max_user_watches=262144 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p;
    #I have used 262144(256K) although the popular option is 524288(512K)
    #I think this eats too much RAM, hence the half size - if it fails with ENOSPC, up it to the higher value as shown below
    #echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p;
myIP=$(ip -4 a | awk '/global/ {print $2}' | cut -d\/ -f1) && printf "Test the install by browsing to http://$myIP:8000 AFTER the next command(s)\n\t.. (you should see the ERPNext login)\n"
cd $HOME/frappe-bench && bench start;    #To Test: point your browser to the relevant URL http://ERPNextIP:8000
                #IF On first load, it looks a bit ugly, just do a refresh of the site (this is a CSS caching problem as far as I can tell)
                #Ctrl-C causes this to exit, and then you can continue as below

# Generate nginx config for your site(s)
bench setup nginx;

sudo rm -f /etc/nginx/sites-enabled/*;    # Remove enable sites and add a symbolic link to nginx conf for frappe
    #sudo ln -s ./config/nginx.conf /etc/nginx/sites-enabled/frappe.conf
    #sudo rm /etc/nginx/sites-enabled/frappe.conf
sudo ln -s $HOME/frappe-bench/config/nginx.conf /etc/nginx/sites-enabled/frappe.conf;
sudo systemctl restart nginx;        # Finally, restart nginx
#______________________________________________________________________________#
bench version;
#    erpnext 12.10.1
#    frappe 12.8.1
#______________________________________________________________________________#

#3) to switch to production
erpnextuser=$(whoami);
sudo bench setup production $erpnextuser;
#The above command creates a linked file at /etc/nginx/conf.d/frappe-bench.conf -> /home/erpnext01/frappe-bench/config/nginx.conf
#which DOES/WILL result in an error
#Jul xx xx:xx:xx d10e nginx[2733]: nginx: [emerg] duplicate upstream "frappe-bench-frappe" in /etc/nginx/sites-enabled/frappe.conf:1
#to fix it, run these commands...
sudo unlink /etc/nginx/conf.d/frappe-bench.conf;
sudo service nginx start;

#bench set-nginx-port $erpnextSiteName 8244;   #Optional IF you want to use a non-standard http:Port
ss -tnlp; #will show you the listeners that are active. You should now also have port 9000,11000,12000,13000 occupied by supervisor processes

bench update;
    # u16 produces this...., but then seems to complete OK??? #my best guess is a python 3.4 vs 3.x mismatch??
    # *** Error compiling '../apps/frappe/frappe/utils/commands.py'... line 11
    #     req = requests.get(f"https://api.github.com/users/{org}/repos", {"type": "sources", "per_page": 200})
    #                                                                 ^
    # SyntaxError: invalid syntax

bench version;
#erpnext 12.10.1
#frappe 12.8.1