STEPS TO REPRODUCE
In setOfCommands.sh file is bunch of commands I used for install of Thingworx, Ignite and Zookeeper.
I wrote this set of commands to backtrack my steps
sudo su
FOLDER=$(pwd)
apt update && apt upgrade -y && apt install -y wget \
mc unzip ca-certificates curl gnupg-agent software-properties-common apt-transport-https \
htop gcc make authbind nfs-common ntp unzip maven && \
wget https://corretto.aws/downloads/latest/amazon-corretto-11-x64-linux-jdk.tar.gz && \
wget https://apache.miloslavbrada.cz/tomcat/tomcat-9/v9.0.41/bin/apache-tomcat-9.0.41.tar.gz && \
wget MED-61111-CD-091_F000_ThingWorx-Platform-Postgres-9-1-0.zip && \
wget https://archive.apache.org/dist/zookeeper/zookeeper-3.5.7/apache-zookeeper-3.5.7-bin.tar.gz && \
# STEP 1:
tar -xvf amazon-corretto-11-x64-linux-jdk.tar.gz
mkdir -p /usr/lib/jdk
mv amazon-corretto-11.0.9.*-linux-x64 /usr/lib/jdk/corretto
update-alternatives --install "/usr/bin/java" "java" "/usr/lib/jdk/corretto/bin/java" 1
update-alternatives --install "/usr/bin/keytool" "keytool" "/usr/lib/jdk/corretto/bin/keytool" 1
chmod a+x /usr/bin/java
chmod a+x /usr/bin/keytool
chown -R root:root /usr/lib/jdk/corretto/
update-alternatives --set java /usr/lib/jdk/corretto/bin/java
update-alternatives --set keytool /usr/lib/jdk/corretto/bin/keytool
java -version
tar -xvf apache-tomcat-9.0.41.tar.gz
mkdir -p /usr/share/tomcat9.0
mv apache-tomcat-9.0.41 /usr/share/tomcat9.0/9.0.41
addgroup --system tomcat9.0 --quiet -force-badname
adduser --system --home /usr/share/tomcat9.0/ --no-create-home --ingroup tomcat9.0 --disabled-password --force-badname --shell /bin/false tomcat9.0
chown -R tomcat9.0:tomcat9.0 /usr/share/tomcat9.0
export JAVA_HOME=/usr/lib/jdk/corretto
export CATALINA_HOME=/usr/share/tomcat9.0/9.0.41
cp /etc/environment /etc/environment.bak
echo "export JAVA_HOME=/usr/lib/jdk/corretto" >> /etc/environment
echo "export CATALINA_HOME=/usr/share/tomcat9.0/9.0.41" >> /etc/environment
echo "export ZOOKEEPER_CONNECTION=192.168.20.250:2181,192.168.20.251:2181,192.168.20.252:2181" >> /etc/environment
source /etc/environment
cd $CATALINA_HOME
chown -Rh tomcat9.0:tomcat9.0 bin/ lib/ webapps/
chmod 775 bin/ lib/ webapps/
chown -R tomcat9.0:tomcat9.0 /usr/share/tomcat9.0/9.0.41
chmod -R 775 /usr/share/tomcat9.0/9.0.41
chown -Rh root:tomcat9.0 conf/
chmod -R 650 conf/
chown -R tomcat9.0:adm logs/ temp/ work/
chmod 760 logs/ temp/ work/
touch /etc/authbind/byport/443
chmod 700 /etc/authbind/byport/443
chown tomcat9.0:tomcat9.0 /etc/authbind/byport/443
echo "0.0.0.0/0:1,1023" > /etc/authbind/byuid/$(id -u tomcat9.0)
sudo chown tomcat9.0:tomcat9.0 /etc/authbind/byuid/$(id -u tomcat9.0)
sudo chmod 700 /etc/authbind/byuid/$(id -u tomcat9.0)
cd $CATALINA_HOME/bin/
tar xvfz commons-daemon-native.tar.gz
cd commons-daemon-*-native-src/unix
./configure --with-java=$JAVA_HOME
make
cp jsvc ../..
rm -rf $CATALINA_HOME/webapps/docs $CATALINA_HOME/webapps/examples $CATALINA_HOME/webapps/host-manager $CATALINA_HOME/webapps/manager $CATALINA_HOME/webapps/ROOT
mkdir /ThingworxStorage /ThingworxBackupStorage /ThingworxPlatform /ThingworxFileRepository
touch /ThingworxPlatform/platform-settings.json
usermod -a -G tomcat9.0 root
cat << "EOF" | sudo tee /etc/systemd/system/tomcat9.0.service
[Unit]
Description=Apache Tomcat Web Application Container
After=network.target
[Service]
Type=forking
PIDFile=/run/tomcat.pid
Environment=CATALINA_PID=/run/tomcat.pid
Environment=JAVA_HOME=/usr/lib/jdk/corretto
Environment=CATALINA_HOME=/usr/share/tomcat9.0/9.0.41
Environment=CATALINA_BASE=/usr/share/tomcat9.0/9.0.41
Environment=CATALINA_OPTS=
Restart=on-failure
RestartSec=5
ExecStart=/usr/share/tomcat9.0/9.0.41/bin/jsvc \
-Dcatalina.home=${CATALINA_HOME} \
-Dcatalina.base=${CATALINA_BASE} \
-Duser.timezone=UTC \
-Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dserver -Dd64 -XX:+UseNUMA \
-XX:+UseG1GC -Dfile.encoding=UTF-8 \
-Djava.library.path=${CATALINA_BASE}/webapps/Thingworx/WEB-INF/extensions \
-cp ${CATALINA_HOME}/bin/commons-daemon.jar:${CATALINA_HOME}/bin/bootstrap.jar:${CATALINA_HOME}/bin/tomcat-juli.jar \
-user tomcat9.0 \
-java-home ${JAVA_HOME} \
-pidfile /run/tomcat.pid \
-errfile ${CATALINA_HOME}/logs/catalina.out \
-outfile ${CATALINA_HOME}/logs/catalina.out \
--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED \
--add-exports=java.base/sun.nio.ch=ALL-UNNAMED \
--add-exports=java.management/com.sun.jmx.mbeanserver=ALL-UNNAMED \
--add-exports=jdk.internal.jvmstat/sun.jvmstat.monitor=ALL-UNNAMED \
--add-exports=java.base/sun.reflect.generics.reflectiveObjects=ALL-UNNAMED \
--add-opens=jdk.management/com.sun.management.internal=ALL-UNNAMED \
--illegal-access=permit \
$CATALINA_OPTS \
org.apache.catalina.startup.Bootstrap
[Install]
WantedBy=multi-user.target
EOF
cd $FOLDER
mkdir tw
unzip MED-61111-CD-091_F000_ThingWorx-Platform-Postgres-9-1-0.zip -d tw
cd tw/
mv Thingworx.war $CATALINA_HOME/webapps
chown tomcat9.0:tomcat9.0 $CATALINA_HOME/webapps/Thingworx.war
chmod 775 $CATALINA_HOME/webapps/Thingworx.war
###
# STEP 2:
$JAVA_HOME/bin/keytool -genkey -alias tomcat9.0 -keyalg RSA -keystore $CATALINA_HOME/conf/.keystore
chown root:tomcat9.0 $CATALINA_HOME/conf/.keystore
chmod 640 $CATALINA_HOME/conf/.keystore
nano $CATALINA_HOME/conf/context.xml
# Uncomment the Manager element in $CATALINA_HOME/conf/context.xml to prevent sessions from persisting across restarts:
#
#
nano $CATALINA_HOME/conf/server.xml
# In the location of the Tomcat installation, open $CATALINA_HOME/conf/server.xml and search for the following line. If found, comment it out and save the file:
#
nano $CATALINA_HOME/conf/tomcat-users.xml
# Define a user in $CATALINA_HOME/conf/tomcat-users.xml
#
nano $CATALINA_HOME/bin/startup.sh
# Comment the following in the file:
# #exec "$PRGDIR"/"$EXECUTABLE" start "$@"
# # Add the following to the end of the file
# exec authbind --deep "$PRGDIR"/"$EXECUTABLE" start "$@"
nano $CATALINA_HOME/conf/web.xml
# Place the following within the web-app tag (after the welcome-file-list tag ). A well-configured web application will override this default in $CATALINA_HOME/webapps/APP_NAME/WEB-INF/web.xml so it won't cause problems.
# java.lang.Throwable/error.jsp
nano $CATALINA_HOME/conf/server.xml
# Add the following inside the tags.
#
systemctl daemon-reload
systemctl enable tomcat9.0.service
#sudo service tomcat9.0 start
cd $CATALINA_HOME/bin
cat << "EOF" | sudo tee setenv.sh
export CATALINA_OPTS="$CATALINA_OPTS -Djava.library.path=/usr/share/tomcat9.0/9.0.xx/webapps/Thingworx/WEB-INF/extensions"
export IGNITE_NO_SHUTDOWN_HOOK=true
export HTTP_PORT=8080
export HTTP_SERVICE_NAME=twaa-tw-01-aio
EOF
chmod +x setenv.sh
systemctl restart tomcat9.0
systemctl status tomcat9.0
###
# Now connect TW server to your NFS / SSHFS server (ThingworxStorage and repository)
# This manual references to NFS
sudo mount 192.168.20.225:/var/nfs/ThingworxStorage /ThingworxStorage
sudo mount 192.168.20.225:/var/nfs/ThingworxFileRepository /ThingworxFileRepository
echo "192.168.20.225:/var/nfs/ThingworxStorage /ThingworxStorage nfs auto,nofail,noatime,nolock,intr,tcp" >> /etc/fstab
echo "192.168.20.225:/var/nfs/ThingworxFileRepository /ThingworxFileRepository nfs auto,nofail,noatime,nolock,intr,tcp" >> /etc/fstab
sudo chown tomcat9.0:tomcat9.0 /ThingworxStorage /ThingworxBackupStorage /ThingworxPlatform
sudo chmod 775 /ThingworxStorage /ThingworxBackupStorage /ThingworxPlatform
###
# Now create platform-settings.json
# Edit it if you wish - PTC support https://support.ptc.com/help/thingworx/platform/r9/en/index.html#page/ThingWorx%2FHelp%2FThingWorxHighAvailability%2Fplatformsettingsforthingworxha.html%23
cat << "EOF" | sudo tee /ThingworxPlatform/platform-settings.json
{
"PlatformSettingsConfig": {
"BasicSettings": {
"BackupStorage":"/ThingworxBackupStorage",
"EnableBackup":false,
"EnableClusteredMode":true,
"EnableSystemLogging":true,
"Storage":"/ThingworxStorage",
"FileRepositoryRoot": "/ThingworxFileRepository"
},
"ContentTypeSettings": {
"supportedMediaEntityContentTypes": [
"image/svg+xml",
"image/png",
"image/gif",
"image/bmp",
"image/jpeg",
"application/pdf",
"image/vnd.microsoft.icon"
]
},
"ExtensionPackageImportPolicy": {
"importEnabled":true,
"allowJarResources":true,
"allowJavascriptResources":true,
"allowCSSResources":true,
"allowJSONResources":true,
"allowWebAppResources":true,
"allowEntities":true,
"allowExtensibleEntities":true,
"haCompatibilityImportLevel":"WARN"
},
"LicensingConnectionSettings": {
"username": "mail",
"password": "password"
},
"ClusteredModeSettings": {
"PlatformId":"twnode0",
"CoordinatorHosts":"localhost:2181,192.168.20.251:2181,192.168.20.252:2181",
"ModelSyncPollInterval":100
},
"AdministratorUserSettings": {
"InitialPassword":"asdfgtrewq"
}
},
"PersistenceProviderPackageConfigs": {
"PostgresPersistenceProviderPackage": {
"ConnectionInformation": {
"jdbcUrl":"jdbc:postgresql://192.168.20.225:5432/thingworx",
"password":"myPassword123",
"username":"twadmin"
}
}
},
"cache": {
"init-timeout":"30 seconds",
"provider-type":"com.thingworx.cache.ignite.IgniteCacheProvider",
"cache-mappings": {},
"ignite": {
"instance-name":"twx-core-server",
"client-mode":true,
"address-resolver": {
"type":"zookeeper",
"connection":"localhost:2181,192.168.20.251:2181,192.168.20.252:2181"
},
"caches": {
"default": {
"cache-mode":"PARTITIONED",
"atomicity-mode":"ATOMIC",
"backups":1
}
}
}
}
}
EOF
### Installing Zookeeper
cd $FOLDER
useradd zookeeper -m
usermod --shell /bin/bash zookeeper
echo "myPassword123" | passwd zookeeper --stdin
usermod -aG sudo zookeeper
sudo getent group sudo
tar -xvf apache-zookeeper-3.5.7-bin.tar.gz
mv apache-zookeeper-3.5.7-bin /opt/zookeeper
sudo mkdir -p /opt/zookeeper/data
chown -R zookeeper:zookeeper /opt/zookeeper/data
cd /opt/zookeeper
touch /opt/zookeeper/confg/zoo.cfg
cat << "EOF" | sudo tee /opt/zookeeper/conf/zoo.cfg
tickTime = 2000
dataDir = /opt/zookeeper/data
clientPort = 2181
initLimit = 5
syncLimit = 2
server.1=192.168.20.250:2888:3888
server.2=192.168.20.251:2888:3888
server.3=192.168.20.252:2888:3888
EOF
chown -R zookeeper:zookeeper /opt/zookeeper
chown -R zookeeper:zookeeper /opt/zookeeper/data
cat << "EOF" | sudo tee /etc/systemd/system/zookeeper.service
[Unit]
Description=Zookeeper Daemon
Documentation=http://zookeeper.apache.org
Requires=network.target
After=network.target
[Service]
Type=forking
WorkingDirectory=/opt/zookeeper
User=zookeeper
Group=zookeeper
ExecStart=/opt/zookeeper/bin/zkServer.sh start /opt/zookeeper/conf/zoo.cfg
ExecStop=/opt/zookeeper/bin/zkServer.sh stop /opt/zookeeper/conf/zoo.cfg
ExecReload=/opt/zookeeper/bin/zkServer.sh restart /opt/zookeeper/conf/zoo.cfg
[Install]
WantedBy=default.target
EOF
echo "1" >> /opt/zookeeper/data/myid ### Change this ID if you are installing onn
### Install Apache Ignite
wget https://archive.apache.org/dist/ignite/2.8.0/apache-ignite-2.8.0-bin.zip
unzip apache-ignite-2.8.0-bin.zip
mv apache-ignite-2.8.0-bin /opt/apache-ignite
echo "export IGNITE_HOME=/opt/apache-ignite" >> /etc/environment
export IGNITE_HOME=/opt/apache-ignite
export IGNITE_WORK=/opt/apache-ignite/work
export IGNITE_WORK_DIR=/opt/apache-ignite/work
echo "export IGNITE_WORK=/opt/apache-ignite/work" >> /etc/environment
echo "export IGNITE_WORK_DIR=/opt/apache-ignite/work" >> /etc/environment
cp MED-61353-CD-091_F000_ThingWorx-Ignite-Cache-Provider-3-6-0.jar $IGNITE_HOME/libs/
cp MED-61353-CD-091_F000_ThingWorx-Ignite-Cache-Provider-API-3-6-0.jar $IGNITE_HOME/libs/
mv $IGNITE_HOME/libs/optional/ignite-zookeeper $IGNITE_HOME/libs/
mv $IGNITE_HOME/libs/optional/ignite-rest-http $IGNITE_HOME/libs/
mv $IGNITE_HOME/libs/optional/ignite-slf4j $IGNITE_HOME/libs/
rm -rf "${IGNITE_HOME}"/libs/ignite-zookeeper/zookeeper-* #&& \
wget -q --no-check-certificate https://repo1.maven.org/maven2/org/apache/zookeeper/zookeeper/3.5.7/zookeeper-3.5.7.jar -P "${IGNITE_HOME}"/libs/ignite-zookeeper/
wget -q --no-check-certificate https://repo1.maven.org/maven2/org/apache/zookeeper/zookeeper-jute/3.5.7/zookeeper-jute-3.5.7.jar -P "${IGNITE_HOME}"/libs/ignite-zookeeper/
wget -q --no-check-certificate https://repo1.maven.org/maven2/io/netty/netty-all/4.1.47.Final/netty-all-4.1.47.Final.jar -P "${IGNITE_HOME}"/libs/ignite-zookeeper/
mv $IGNITE_HOME/config/default-config.xml $IGNITE_HOME/config/default-config.xml.bak
touch $IGNITE_HOME/config/default-config.xml
nano $IGNITE_HOME/config/default-config.xml
cat << "EOF" | sudo tee /opt/apache-ignite/config/default-config.xml
EOF
cat << "EOF" | sudo tee /lib/systemd/system/ignite.service
[Unit]
Description=Apache Ignite Service
After=network.target
[Service]
WorkingDirectory=/opt/apache-ignite
User=root
PrivateDevices=yes
ProtectSystem=full
Type=simple
ExecReload=/bin/kill -HUP $MAINPID
KillMode=mixed
KillSignal=SIGTERM
TimeoutStopSec=10
ExecStart=/opt/apache-ignite/bin/ignite.sh
SyslogIdentifier=Ignite
Restart=on-failure
RestartSec=5
[Install]
WantedBy=multi-user.target
Alias=ignite.service
EOF
sudo systemctl daemon-reload
sudo systemctl enable ignite.service
sudo systemctl enable zookeeper.service
sudo systemctl enable tomcat9.0.service
sudo service zookeeper start
sudo service ignite start
sudo service tomcat9.0 start
------------
Greetings!
I'm installing ThingWorx Platform 9.1 in clustered (active-active) mode and I can't make it work. My Ignite 2.8.0 service is running and also Zookeeper 3.5.7. But when I try to run Tomcat 9.0.41 it loads and after a few seconds it crash. As you can see in the provided log problem is in ConsistentId null. I don't know why ThingWorx cannot connect to Ignite through Zookeeper. All of the instances are running on one server. However it wasn't working even on dedicated servers. The system I'm running is fully upgraded Ubuntu 20.04 and Java is OpenJDK Runtime Environment Corretto-11.0.9.12.1. I also run Zookeeper, Ignite and Tomcat as services. I exported IGNITE_HOME and IGNITE_WORK into /etc/environment for sure. Tcpdump shows no activity after starting tomcat so I assume that Thingworx is not connected even to Zookeeper. I even try to set -J-DIGNITE_JETTY_PORT=9999 and admin.enableServer=false so port 8080 is free for Tomcat but it didnt help. Could you help me with that? Thank you.
Guides I used:
PTC guides
https://ignite.apache.org/docs/latest/installation/installing-using-zip
------------
Thingworx node cannot log to Ignite and Ignite have ConsistentId null
------------
2020-12-10 11:08:29.294+0000 [L: INFO] [O: c.t.c.i.IgniteCacheProvider] [I: ] [U: ] [S: ] [P: twnode0] [T: ForkJoinPool.commonPool-worker-3] Starting Ignite ConsistentId null with configuration: IgniteConfiguration [igniteInstanceName=twx-core-server, pubPoolSize=8, svcPoolSize=null, callbackPoolSize=8, stripedPoolSize=8, sysPoolSize=8, mgmtPoolSize=4, igfsPoolSize=3, dataStreamerPoolSize=8, utilityCachePoolSize=8, utilityCacheKeepAliveTime=60000, p2pPoolSize=2, qryPoolSize=8, sqlQryHistSize=1000, dfltQryTimeout=0, igniteHome=null, igniteWorkDir=null, mbeanSrv=null, nodeId=null, marsh=null, marshLocJobs=false, daemon=false, p2pEnabled=true, netTimeout=5000, netCompressionLevel=1, sndRetryDelay=1000, sndRetryCnt=3, metricsHistSize=10000, metricsUpdateFreq=2000, metricsExpTime=9223372036854775807, discoSpi=TcpDiscoverySpi [addrRslvr=null, sockTimeout=0, ackTimeout=0, marsh=null, reconCnt=10, reconDelay=2000, maxAckTimeout=600000, soLinger=5, forceSrvMode=false, clientReconnectDisabled=false, internalLsnr=null, skipAddrsRandomization=false], segPlc=STOP, segResolveAttempts=2, waitForSegOnStart=true, allResolversPassReq=true, segChkFreq=10000, commSpi=null, evtSpi=null, colSpi=null, deploySpi=null, indexingSpi=null, addrRslvr=null, encryptionSpi=null, clientMode=true, rebalanceThreadPoolSize=4, rebalanceTimeout=10000, rebalanceBatchesPrefetchCnt=3, rebalanceThrottle=0, rebalanceBatchSize=524288, txCfg=TransactionConfiguration [txSerEnabled=false, dfltIsolation=REPEATABLE_READ, dfltConcurrency=PESSIMISTIC, dfltTxTimeout=0, txTimeoutOnPartitionMapExchange=0, deadlockTimeout=10000, pessimisticTxLogSize=0, pessimisticTxLogLinger=10000, tmLookupClsName=null, txManagerFactory=null, useJtaSync=false], cacheSanityCheckEnabled=true, discoStartupDelay=60000, deployMode=SHARED, p2pMissedCacheSize=100, locHost=null, timeSrvPortBase=31100, timeSrvPortRange=100, failureDetectionTimeout=10000, sysWorkerBlockedTimeout=null, clientFailureDetectionTimeout=30000, metricsLogFreq=0, hadoopCfg=null, connectorCfg=ConnectorConfiguration [jettyPath=null, host=null, port=11211, noDelay=true, directBuf=false, sndBufSize=32768, rcvBufSize=32768, idleQryCurTimeout=600000, idleQryCurCheckFreq=60000, sndQueueLimit=0, selectorCnt=3, idleTimeout=7000, sslEnabled=false, sslClientAuth=false, sslCtxFactory=null, sslFactory=null, portRange=100, threadPoolSize=8, msgInterceptor=null], odbcCfg=null, warmupClos=null, atomicCfg=AtomicConfiguration [seqReserveSize=1000, cacheMode=REPLICATED, backups=0, aff=null, grpName=null], classLdr=null, sslCtxFactory=null, platformCfg=null, binaryCfg=null, memCfg=null, pstCfg=null, dsCfg=null, activeOnStart=true, autoActivation=true, longQryWarnTimeout=3000, sqlConnCfg=null, cliConnCfg=ClientConnectorConfiguration [host=null, port=10800, portRange=100, sockSndBufSize=0, sockRcvBufSize=0, tcpNoDelay=true, maxOpenCursorsPerConn=128, threadPoolSize=8, idleTimeout=0, handshakeTimeout=10000, jdbcEnabled=true, odbcEnabled=true, thinCliEnabled=true, sslEnabled=false, useIgniteSslCtxFactory=true, sslClientAuth=false, sslCtxFactory=null, thinCliCfg=ThinClientConfiguration [maxActiveTxPerConn=100]], mvccVacuumThreadCnt=2, mvccVacuumFreq=5000, authEnabled=false, failureHnd=null, commFailureRslvr=null]
2020-12-10 11:08:29.373+0000 [L: ERROR] [O: c.t.s.ThingWorxBootstrapper] [I: ] [U: SuperUser] [S: ] [P: twnode0] [T: main] *** CRITICAL ERROR ON STARTUP: Failed to start CacheProvider com.thingworx.cache.ignite.IgniteCacheProvider
2020-12-10 11:08:29.375+0000 [L: ERROR] [O: c.t.s.ThingWorxBootstrapper] [I: ] [U: SuperUser] [S: ] [P: twnode0] [T: main] *** Web Application STATE is being set to ERROR! ***
2020-12-10 11:08:29.377+0000 [L: INFO] [O: c.t.s.ThingWorxServer] [I: ] [U: SuperUser] [S: ] [P: twnode0] [T: main] >>>>>>> PLATFORM SHUTDOWN START <<<<<<<<<