INFO [2018-09-07 12:16:44,393] org.eclipse.jetty.util.log: Logging initialized @2305ms INFO [2018-09-07 12:16:44,481] io.dropwizard.server.DefaultServerFactory: Registering jersey handler with root path prefix: / INFO [2018-09-07 12:16:44,481] io.dropwizard.server.DefaultServerFactory: Registering admin handler with root path prefix: / INFO [2018-09-07 12:16:44,481] io.dropwizard.assets.AssetsBundle: Registering AssetBundle with name: swagger for path /swagger/* INFO [2018-09-07 12:16:44,520] io.dropwizard.assets.AssetsBundle: Registering AssetBundle with name: swagger-assets for path /swagger-static/* INFO [2018-09-07 12:16:51,918] org.reflections.Reflections: Reflections took 7375 ms to scan 1 urls, producing 108856 keys and 249936 values INFO [2018-09-07 12:16:52,495] com.thingworx.analytics.ccc.dropwizard.utils.DropwizardBaseApplication: BEGIN database flyway migrations INFO [2018-09-07 12:16:52,504] org.flywaydb.core.internal.util.VersionPrinter: Flyway 4.0.3 by Boxfuse INFO [2018-09-07 12:16:52,674] org.flywaydb.core.internal.dbsupport.DbSupportFactory: Database: jdbc:h2:mem:/data/training/db:training (H2 1.4) INFO [2018-09-07 12:16:52,795] org.flywaydb.core.internal.command.DbValidate: Successfully validated 2 migrations (execution time 00:00.008s) INFO [2018-09-07 12:16:52,807] org.flywaydb.core.internal.metadatatable.MetaDataTableImpl: Creating Metadata table: "PUBLIC"."schema_version" INFO [2018-09-07 12:16:52,824] org.flywaydb.core.internal.command.DbMigrate: Current version of schema "PUBLIC": << Empty Schema >> INFO [2018-09-07 12:16:52,824] org.flywaydb.core.internal.command.DbMigrate: Migrating schema "PUBLIC" to version 1 - Create Base Schema INFO [2018-09-07 12:16:52,851] org.flywaydb.core.internal.command.DbMigrate: Migrating schema "PUBLIC" to version 1.2 - Create Tag Table INFO [2018-09-07 12:16:52,865] org.flywaydb.core.internal.command.DbMigrate: Successfully applied 2 migrations to schema "PUBLIC" (execution time 00:00.059s). INFO [2018-09-07 12:16:52,867] com.thingworx.analytics.ccc.dropwizard.utils.DropwizardBaseApplication: END database flyway migrations INFO [2018-09-07 12:16:52,870] io.dropwizard.server.DefaultServerFactory: Registering jersey handler with root path prefix: / INFO [2018-09-07 12:16:52,870] io.dropwizard.server.DefaultServerFactory: Registering admin handler with root path prefix: / INFO [2018-09-07 12:16:52,878] io.dropwizard.server.ServerFactory: Starting ThingWorx Analytics Training Microservice ..... . ... .H8888888h. ~-. .uef^" @88> .zf"` `"tu 888888888888x `> :d88E %8P u. u. x88 '8N. .u . u. u. .u . X~ `?888888hx~ `888E . x@88k u@88c. uL 888k d88& .u u .d88B :@8c x@88k u@88c. .u .d88B :@8c ' x8.^"*88*" 888E .z8k .@88u ^"8888""8888" .ue888Nc.. 8888N. @888F ud8888. us888u. ="8888f8888r ^"8888""8888" ud8888. ="8888f8888r `-:- X8888x 888E~?888L ''888E` 8888 888R d88E`"888E` `88888 9888% :888'8888. .@88 "8888" 4888>'88" 8888 888R :888'8888. 4888>'88" 488888> 888E 888E 888E 8888 888R 888E 888E %888 "88F d888 '88%" 9888 9888 4888> ' 8888 888R d888 '88%" 4888> ' .. `"88* 888E 888E 888E 8888 888R 888E 888E 8" "*h=~ 8888.+" 9888 9888 4888> 8888 888R 8888.+" 4888> x88888nX" . 888E 888E 888E 8888 888R 888E 888E z8Weu 8888L 9888 9888 .d888L .+ 8888 888R 8888L .d888L .+ !"*8888888n.. : 888E 888E 888& "*88*" 8888" 888& .888E ""88888i. Z '8888c. .+ 9888 9888 ^"8888*" "*88*" 8888" '8888c. .+ ^"8888*" ' "*88888888* m888N= 888> R888" "" 'Y" *888" 888& " "8888888* "88888% "888*""888" "Y" "" 'Y" "88888% "Y" ^"***"` `Y" 888 "" `" "888E ^"**"" "YP' ^Y" ^Y' "YP' J88" .dWi `88E @% 4888~ J8% :" ^"===*"` INFO [2018-09-07 12:16:52,965] org.eclipse.jetty.setuid.SetUIDListener: Opened application@672a7893{HTTP/1.1,[http/1.1]}{0.0.0.0:8090} INFO [2018-09-07 12:16:52,966] org.eclipse.jetty.setuid.SetUIDListener: Opened admin@2e18c730{HTTP/1.1,[http/1.1]}{0.0.0.0:8091} INFO [2018-09-07 12:16:52,967] org.eclipse.jetty.server.Server: jetty-9.3.z-SNAPSHOT INFO [2018-09-07 12:16:53,581] io.dropwizard.jersey.DropwizardResourceConfig: The following paths were found for the configured resources: GET /swagger (io.federecio.dropwizard.swagger.SwaggerResource) GET /swagger.{type:json|yaml} (io.swagger.jaxrs.listing.ApiListingResource) GET /training (com.thingworx.analytics.training.resources.JobExecInfoResource) POST /training (com.thingworx.analytics.training.resources.TrainingResource) GET /training/configs (com.thingworx.analytics.training.resources.TrainingConfigsResource) GET /training/status (com.thingworx.analytics.training.resources.JobExecInfoResource) DELETE /training/{jobId} (com.thingworx.analytics.training.resources.JobExecInfoResource) GET /training/{jobId} (com.thingworx.analytics.training.resources.JobExecInfoResource) GET /training/{jobId}/configs (com.thingworx.analytics.training.resources.TrainingConfigsResource) GET /training/{jobId}/status (com.thingworx.analytics.training.resources.JobExecInfoResource) DELETE /training/{jobId}/tags (com.thingworx.analytics.training.resources.TagsResource) GET /training/{jobId}/tags (com.thingworx.analytics.training.resources.TagsResource) POST /training/{jobId}/tags (com.thingworx.analytics.training.resources.TagsResource) PUT /training/{jobId}/tags (com.thingworx.analytics.training.resources.TagsResource) INFO [2018-09-07 12:16:53,583] org.eclipse.jetty.server.handler.ContextHandler: Started i.d.j.MutableServletContextHandler@7e4d2615{/,null,AVAILABLE} INFO [2018-09-07 12:16:53,588] io.dropwizard.setup.AdminEnvironment: tasks = POST /tasks/log-level (io.dropwizard.servlets.tasks.LogConfigurationTask) POST /tasks/gc (io.dropwizard.servlets.tasks.GarbageCollectionTask) INFO [2018-09-07 12:16:53,593] org.eclipse.jetty.server.handler.ContextHandler: Started i.d.j.MutableServletContextHandler@5a905a16{/,null,AVAILABLE} INFO [2018-09-07 12:16:53,600] org.eclipse.jetty.server.AbstractConnector: Started application@672a7893{HTTP/1.1,[http/1.1]}{0.0.0.0:8090} INFO [2018-09-07 12:16:53,605] org.eclipse.jetty.server.AbstractConnector: Started admin@2e18c730{HTTP/1.1,[http/1.1]}{0.0.0.0:8091} INFO [2018-09-07 12:16:53,605] org.eclipse.jetty.server.Server: Started @11519ms INFO [2018-09-07 12:23:39,241] com.thingworx.analytics.ccc.data.datasources.JsonLocalDatasetSource: Received [508] rows to score WARN [2018-09-07 12:23:39,352] com.thingworx.analytics.ccc.data.spark.SparkSupplier: No container XML defined so there will be no SparkSupplier. INFO [2018-09-07 12:23:40,602] org.spark_project.jetty.util.log: Logging initialized @418515ms INFO [2018-09-07 12:23:42,160] com.thingworx.analytics.training.Learner: Begin training Learner [trainer=MultiGoalTrainer [Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0]], transformer=DoNothingTransformer []] INFO [2018-09-07 12:23:42,170] com.thingworx.analytics.training.Learner: Begin training Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0] INFO [2018-09-07 12:23:42,180] com.thingworx.analytics.training.ensemble.AbstractEnsembleModel: Training ensemble SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]] INFO [2018-09-07 12:23:42,196] com.thingworx.analytics.training.Learner: Begin training Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]] INFO [2018-09-07 12:23:42,212] com.thingworx.analytics.signals.SignalAnalyzer: Getting signals 1 at a time. WARN [2018-09-07 12:23:42,455] org.apache.spark.executor.Executor: 1 block locks were not released by TID = 29: [rdd_26_0] WARN [2018-09-07 12:23:42,476] org.apache.spark.executor.Executor: 1 block locks were not released by TID = 30: [rdd_26_0] WARN [2018-09-07 12:23:42,811] com.github.fommil.netlib.BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeSystemBLAS WARN [2018-09-07 12:23:42,811] com.github.fommil.netlib.BLAS: Failed to load implementation from: com.github.fommil.netlib.NativeRefBLAS INFO [2018-09-07 12:23:43,514] breeze.optimize.LBFGS: Step Size: 17.25 INFO [2018-09-07 12:23:43,515] breeze.optimize.LBFGS: Val and Grad Norm: 0.0872149 (rel: 0.195) 0.0222469 INFO [2018-09-07 12:23:43,564] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:43,564] breeze.optimize.LBFGS: Val and Grad Norm: 0.0833642 (rel: 0.0442) 0.0130698 INFO [2018-09-07 12:23:43,607] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:43,607] breeze.optimize.LBFGS: Val and Grad Norm: 0.0791860 (rel: 0.0501) 0.0180953 INFO [2018-09-07 12:23:43,646] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:43,646] breeze.optimize.LBFGS: Val and Grad Norm: 0.0683841 (rel: 0.136) 0.0528808 INFO [2018-09-07 12:23:43,683] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:43,683] breeze.optimize.LBFGS: Val and Grad Norm: 0.0541164 (rel: 0.209) 0.0577147 INFO [2018-09-07 12:23:43,715] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:43,715] breeze.optimize.LBFGS: Val and Grad Norm: 0.0153602 (rel: 0.716) 0.0269260 INFO [2018-09-07 12:23:43,745] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:43,745] breeze.optimize.LBFGS: Val and Grad Norm: 0.00842328 (rel: 0.452) 0.0153666 INFO [2018-09-07 12:23:43,798] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.5339274348847078 fval: 0.0057532434725897025 rhs: 0.008422821848555832 cdd: -7.750772275909898E-5 INFO [2018-09-07 12:23:43,798] breeze.optimize.LBFGS: Step Size: 0.5339 INFO [2018-09-07 12:23:43,798] breeze.optimize.LBFGS: Val and Grad Norm: 0.00575324 (rel: 0.317) 0.00547488 INFO [2018-09-07 12:23:43,822] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:43,823] breeze.optimize.LBFGS: Val and Grad Norm: 0.00531785 (rel: 0.0757) 0.00468987 INFO [2018-09-07 12:23:43,846] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:43,846] breeze.optimize.LBFGS: Val and Grad Norm: 0.00412863 (rel: 0.224) 0.00229590 INFO [2018-09-07 12:23:43,869] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:43,869] breeze.optimize.LBFGS: Val and Grad Norm: 0.00323559 (rel: 0.216) 0.00224546 INFO [2018-09-07 12:23:43,916] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.7179197920692222 fval: 0.0016635840929767702 rhs: 0.0032353721285024615 cdd: -4.7517202237422936E-4 INFO [2018-09-07 12:23:43,916] breeze.optimize.LBFGS: Step Size: 0.7179 INFO [2018-09-07 12:23:43,916] breeze.optimize.LBFGS: Val and Grad Norm: 0.00166358 (rel: 0.486) 0.00161917 INFO [2018-09-07 12:23:43,938] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:43,939] breeze.optimize.LBFGS: Val and Grad Norm: 0.00152114 (rel: 0.0856) 0.00203881 INFO [2018-09-07 12:23:43,963] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:43,963] breeze.optimize.LBFGS: Val and Grad Norm: 0.00151996 (rel: 0.000780) 0.00233203 INFO [2018-09-07 12:23:43,989] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:43,990] breeze.optimize.LBFGS: Val and Grad Norm: 0.00142359 (rel: 0.0634) 0.000310109 INFO [2018-09-07 12:23:44,020] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:44,020] breeze.optimize.LBFGS: Val and Grad Norm: 0.00142183 (rel: 0.00124) 0.000105378 INFO [2018-09-07 12:23:44,042] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:44,042] breeze.optimize.LBFGS: Val and Grad Norm: 0.00142096 (rel: 0.000610) 0.000101045 INFO [2018-09-07 12:23:44,066] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:44,067] breeze.optimize.LBFGS: Val and Grad Norm: 0.00141604 (rel: 0.00347) 0.000132198 INFO [2018-09-07 12:23:44,089] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:44,090] breeze.optimize.LBFGS: Val and Grad Norm: 0.00140727 (rel: 0.00619) 0.000119716 INFO [2018-09-07 12:23:44,202] breeze.optimize.LBFGS: Step Size: 5.063 INFO [2018-09-07 12:23:44,202] breeze.optimize.LBFGS: Val and Grad Norm: 0.000870126 (rel: 0.382) 0.000895931 INFO [2018-09-07 12:23:44,245] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.21712869200017648 fval: 8.693272280371541E-4 rhs: 8.701205659878397E-4 cdd: 1.1688396091450847E-4 INFO [2018-09-07 12:23:44,245] breeze.optimize.LBFGS: Step Size: 0.2171 INFO [2018-09-07 12:23:44,246] breeze.optimize.LBFGS: Val and Grad Norm: 0.000869327 (rel: 0.000918) 0.00121560 INFO [2018-09-07 12:23:44,266] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:44,266] breeze.optimize.LBFGS: Val and Grad Norm: 0.000787992 (rel: 0.0936) 0.000182832 INFO [2018-09-07 12:23:44,295] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:44,295] breeze.optimize.LBFGS: Val and Grad Norm: 0.000773229 (rel: 0.0187) 0.000665353 INFO [2018-09-07 12:23:44,319] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:44,320] breeze.optimize.LBFGS: Val and Grad Norm: 0.000767542 (rel: 0.00735) 0.000215142 INFO [2018-09-07 12:23:44,341] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:44,341] breeze.optimize.LBFGS: Val and Grad Norm: 0.000765876 (rel: 0.00217) 2.93142e-05 INFO [2018-09-07 12:23:44,360] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:44,360] breeze.optimize.LBFGS: Val and Grad Norm: 0.000765502 (rel: 0.000489) 6.01434e-05 INFO [2018-09-07 12:23:44,381] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:44,381] breeze.optimize.LBFGS: Val and Grad Norm: 0.000765141 (rel: 0.000472) 8.79669e-05 INFO [2018-09-07 12:23:44,399] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:44,399] breeze.optimize.LBFGS: Val and Grad Norm: 0.000763789 (rel: 0.00177) 0.000150806 INFO [2018-09-07 12:23:44,418] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:44,418] breeze.optimize.LBFGS: Val and Grad Norm: 0.000760792 (rel: 0.00392) 0.000202496 INFO [2018-09-07 12:23:44,435] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:44,436] breeze.optimize.LBFGS: Val and Grad Norm: 0.000752398 (rel: 0.0110) 0.000124886 INFO [2018-09-07 12:23:44,470] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.4074114408098871 fval: 7.500516244690026E-4 rhs: 7.523980429416441E-4 cdd: 1.1232237128947016E-6 INFO [2018-09-07 12:23:44,470] breeze.optimize.LBFGS: Step Size: 0.4074 INFO [2018-09-07 12:23:44,470] breeze.optimize.LBFGS: Val and Grad Norm: 0.000750052 (rel: 0.00312) 0.000111481 INFO [2018-09-07 12:23:44,475] com.thingworx.analytics.training.Learner: Finished training Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]] after: {} INFO [2018-09-07 12:23:44,483] com.thingworx.analytics.training.Learner: Finished training Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0] after: {} INFO [2018-09-07 12:23:44,485] com.thingworx.analytics.training.Learner: Finished training Learner [trainer=MultiGoalTrainer [Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0]], transformer=DoNothingTransformer []] after: {} INFO [2018-09-07 12:23:44,828] com.thingworx.analytics.training.Learner: Begin training Learner [trainer=MultiGoalTrainer [Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0]], transformer=DoNothingTransformer []] INFO [2018-09-07 12:23:44,830] com.thingworx.analytics.training.Learner: Begin training Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0] INFO [2018-09-07 12:23:44,832] com.thingworx.analytics.training.ensemble.AbstractEnsembleModel: Training ensemble SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]] INFO [2018-09-07 12:23:44,836] com.thingworx.analytics.training.Learner: Begin training Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]] INFO [2018-09-07 12:23:44,837] com.thingworx.analytics.signals.SignalAnalyzer: Getting signals 1 at a time. WARN [2018-09-07 12:23:44,949] org.apache.spark.executor.Executor: 1 block locks were not released by TID = 183: [rdd_101_0] WARN [2018-09-07 12:23:44,962] org.apache.spark.executor.Executor: 1 block locks were not released by TID = 184: [rdd_101_0] INFO [2018-09-07 12:23:45,063] breeze.optimize.StrongWolfeLineSearch: Line search t: 6.6324867117633985 fval: 0.0851967688602477 rhs: 0.08943522411582058 cdd: -1.5685398900729228E-4 INFO [2018-09-07 12:23:45,063] breeze.optimize.LBFGS: Step Size: 6.632 INFO [2018-09-07 12:23:45,064] breeze.optimize.LBFGS: Val and Grad Norm: 0.0851968 (rel: 0.0474) 0.0129440 INFO [2018-09-07 12:23:45,085] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,085] breeze.optimize.LBFGS: Val and Grad Norm: 0.0838092 (rel: 0.0163) 0.0151939 INFO [2018-09-07 12:23:45,110] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,111] breeze.optimize.LBFGS: Val and Grad Norm: 0.0735525 (rel: 0.122) 0.0868813 INFO [2018-09-07 12:23:45,136] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,139] breeze.optimize.LBFGS: Val and Grad Norm: 0.0612876 (rel: 0.167) 0.0721551 INFO [2018-09-07 12:23:45,232] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,233] breeze.optimize.LBFGS: Val and Grad Norm: 0.0278407 (rel: 0.546) 0.0615733 INFO [2018-09-07 12:23:45,268] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,269] breeze.optimize.LBFGS: Val and Grad Norm: 0.00782814 (rel: 0.719) 0.0161016 INFO [2018-09-07 12:23:45,293] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,293] breeze.optimize.LBFGS: Val and Grad Norm: 0.00521191 (rel: 0.334) 0.00531923 INFO [2018-09-07 12:23:45,316] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,316] breeze.optimize.LBFGS: Val and Grad Norm: 0.00447149 (rel: 0.142) 0.00424730 INFO [2018-09-07 12:23:45,336] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,336] breeze.optimize.LBFGS: Val and Grad Norm: 0.00381559 (rel: 0.147) 0.00421387 INFO [2018-09-07 12:23:45,356] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,356] breeze.optimize.LBFGS: Val and Grad Norm: 0.00307558 (rel: 0.194) 0.00389646 INFO [2018-09-07 12:23:45,376] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,376] breeze.optimize.LBFGS: Val and Grad Norm: 0.00291754 (rel: 0.0514) 0.00137637 INFO [2018-09-07 12:23:45,397] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,398] breeze.optimize.LBFGS: Val and Grad Norm: 0.00271304 (rel: 0.0701) 0.000870046 INFO [2018-09-07 12:23:45,418] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,418] breeze.optimize.LBFGS: Val and Grad Norm: 0.00227641 (rel: 0.161) 0.000371143 INFO [2018-09-07 12:23:45,441] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,441] breeze.optimize.LBFGS: Val and Grad Norm: 0.00214821 (rel: 0.0563) 0.000344501 INFO [2018-09-07 12:23:45,499] breeze.optimize.LBFGS: Step Size: 1.500 INFO [2018-09-07 12:23:45,499] breeze.optimize.LBFGS: Val and Grad Norm: 0.00163901 (rel: 0.237) 0.00342290 INFO [2018-09-07 12:23:45,525] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,525] breeze.optimize.LBFGS: Val and Grad Norm: 0.00150800 (rel: 0.0799) 0.000317612 INFO [2018-09-07 12:23:45,571] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.3664242817061938 fval: 0.0014810209600191033 rhs: 0.0015079915561071896 cdd: 2.217513698137131E-6 INFO [2018-09-07 12:23:45,572] breeze.optimize.LBFGS: Step Size: 0.3664 INFO [2018-09-07 12:23:45,572] breeze.optimize.LBFGS: Val and Grad Norm: 0.00148102 (rel: 0.0179) 0.000868763 INFO [2018-09-07 12:23:45,600] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,601] breeze.optimize.LBFGS: Val and Grad Norm: 0.00147153 (rel: 0.00641) 0.000263753 INFO [2018-09-07 12:23:45,627] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,627] breeze.optimize.LBFGS: Val and Grad Norm: 0.00146397 (rel: 0.00514) 8.99399e-05 INFO [2018-09-07 12:23:45,653] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,654] breeze.optimize.LBFGS: Val and Grad Norm: 0.00146086 (rel: 0.00212) 9.98368e-05 INFO [2018-09-07 12:23:45,803] breeze.optimize.LBFGS: Step Size: 7.594 INFO [2018-09-07 12:23:45,803] breeze.optimize.LBFGS: Val and Grad Norm: 0.00136725 (rel: 0.0641) 0.00112747 INFO [2018-09-07 12:23:45,826] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,826] breeze.optimize.LBFGS: Val and Grad Norm: 0.00134023 (rel: 0.0198) 0.00243925 INFO [2018-09-07 12:23:45,879] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.2499704356382354 fval: 0.0012496079183834691 rhs: 0.0013402172025689658 cdd: -2.700813241739517E-4 INFO [2018-09-07 12:23:45,880] breeze.optimize.LBFGS: Step Size: 0.2500 INFO [2018-09-07 12:23:45,880] breeze.optimize.LBFGS: Val and Grad Norm: 0.00124961 (rel: 0.0676) 0.00326102 INFO [2018-09-07 12:23:45,929] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.1350251613962341 fval: 0.0011735313775000265 rhs: 0.0012495920331760352 cdd: 2.8071999544373744E-4 INFO [2018-09-07 12:23:45,929] breeze.optimize.LBFGS: Step Size: 0.1350 INFO [2018-09-07 12:23:45,929] breeze.optimize.LBFGS: Val and Grad Norm: 0.00117353 (rel: 0.0609) 0.00417789 INFO [2018-09-07 12:23:45,954] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,954] breeze.optimize.LBFGS: Val and Grad Norm: 0.000992441 (rel: 0.154) 0.000794672 INFO [2018-09-07 12:23:45,978] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:45,978] breeze.optimize.LBFGS: Val and Grad Norm: 0.000963005 (rel: 0.0297) 0.000482779 INFO [2018-09-07 12:23:46,002] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,002] breeze.optimize.LBFGS: Val and Grad Norm: 0.000920232 (rel: 0.0444) 0.000290187 INFO [2018-09-07 12:23:46,023] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,023] breeze.optimize.LBFGS: Val and Grad Norm: 0.000893465 (rel: 0.0291) 0.000197381 INFO [2018-09-07 12:23:46,041] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,042] breeze.optimize.LBFGS: Val and Grad Norm: 0.000873658 (rel: 0.0222) 0.000215955 INFO [2018-09-07 12:23:46,084] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.67943329156808 fval: 8.462930662501329E-4 rhs: 8.73654095237577E-4 cdd: -1.338195080293519E-6 INFO [2018-09-07 12:23:46,085] breeze.optimize.LBFGS: Step Size: 0.6794 INFO [2018-09-07 12:23:46,085] breeze.optimize.LBFGS: Val and Grad Norm: 0.000846293 (rel: 0.0313) 0.00100228 INFO [2018-09-07 12:23:46,105] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,105] breeze.optimize.LBFGS: Val and Grad Norm: 0.000816021 (rel: 0.0358) 0.000768158 INFO [2018-09-07 12:23:46,123] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,124] breeze.optimize.LBFGS: Val and Grad Norm: 0.000780296 (rel: 0.0438) 0.000825636 INFO [2018-09-07 12:23:46,141] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,142] breeze.optimize.LBFGS: Val and Grad Norm: 0.000762152 (rel: 0.0233) 0.000328354 INFO [2018-09-07 12:23:46,162] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,162] breeze.optimize.LBFGS: Val and Grad Norm: 0.000751933 (rel: 0.0134) 0.000384099 INFO [2018-09-07 12:23:46,198] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.46432702557702255 fval: 7.369041279202201E-4 rhs: 7.519306595429208E-4 cdd: -1.5373928927543776E-7 INFO [2018-09-07 12:23:46,198] breeze.optimize.LBFGS: Step Size: 0.4643 INFO [2018-09-07 12:23:46,198] breeze.optimize.LBFGS: Val and Grad Norm: 0.000736904 (rel: 0.0200) 0.00150702 INFO [2018-09-07 12:23:46,229] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,229] breeze.optimize.LBFGS: Val and Grad Norm: 0.000718030 (rel: 0.0256) 0.000641050 INFO [2018-09-07 12:23:46,250] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,250] breeze.optimize.LBFGS: Val and Grad Norm: 0.000709099 (rel: 0.0124) 0.000506545 INFO [2018-09-07 12:23:46,276] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,276] breeze.optimize.LBFGS: Val and Grad Norm: 0.000705661 (rel: 0.00485) 0.000214939 INFO [2018-09-07 12:23:46,301] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,301] breeze.optimize.LBFGS: Val and Grad Norm: 0.000701934 (rel: 0.00528) 0.000181362 INFO [2018-09-07 12:23:46,327] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,327] breeze.optimize.LBFGS: Val and Grad Norm: 0.000692927 (rel: 0.0128) 0.000377323 INFO [2018-09-07 12:23:46,353] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,353] breeze.optimize.LBFGS: Val and Grad Norm: 0.000690606 (rel: 0.00335) 0.000854958 INFO [2018-09-07 12:23:46,373] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,373] breeze.optimize.LBFGS: Val and Grad Norm: 0.000687201 (rel: 0.00493) 0.000111079 INFO [2018-09-07 12:23:46,392] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,392] breeze.optimize.LBFGS: Val and Grad Norm: 0.000685823 (rel: 0.00201) 0.000175441 INFO [2018-09-07 12:23:46,413] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,413] breeze.optimize.LBFGS: Val and Grad Norm: 0.000684687 (rel: 0.00166) 0.000328264 INFO [2018-09-07 12:23:46,434] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:46,434] breeze.optimize.LBFGS: Val and Grad Norm: 0.000683807 (rel: 0.00128) 9.48059e-05 INFO [2018-09-07 12:23:46,434] com.thingworx.analytics.training.Learner: Finished training Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]] after: {} INFO [2018-09-07 12:23:46,435] com.thingworx.analytics.training.Learner: Finished training Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0] after: {} INFO [2018-09-07 12:23:46,435] com.thingworx.analytics.training.Learner: Finished training Learner [trainer=MultiGoalTrainer [Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0]], transformer=DoNothingTransformer []] after: {} INFO [2018-09-07 12:23:46,705] com.thingworx.analytics.training.Learner: Begin training Learner [trainer=MultiGoalTrainer [Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0]], transformer=DoNothingTransformer []] INFO [2018-09-07 12:23:46,706] com.thingworx.analytics.training.Learner: Begin training Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0] INFO [2018-09-07 12:23:46,708] com.thingworx.analytics.training.ensemble.AbstractEnsembleModel: Training ensemble SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]] INFO [2018-09-07 12:23:46,712] com.thingworx.analytics.training.Learner: Begin training Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]] INFO [2018-09-07 12:23:46,714] com.thingworx.analytics.signals.SignalAnalyzer: Getting signals 1 at a time. WARN [2018-09-07 12:23:46,877] org.apache.spark.executor.Executor: 1 block locks were not released by TID = 391: [rdd_194_0] WARN [2018-09-07 12:23:46,896] org.apache.spark.executor.Executor: 1 block locks were not released by TID = 392: [rdd_194_0] INFO [2018-09-07 12:23:46,978] breeze.optimize.LBFGS: Step Size: 40.08 INFO [2018-09-07 12:23:46,978] breeze.optimize.LBFGS: Val and Grad Norm: 0.0797714 (rel: 0.0825) 0.0482342 INFO [2018-09-07 12:23:47,013] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,014] breeze.optimize.LBFGS: Val and Grad Norm: 0.0680020 (rel: 0.148) 0.0331069 INFO [2018-09-07 12:23:47,043] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,043] breeze.optimize.LBFGS: Val and Grad Norm: 0.0487710 (rel: 0.283) 0.0493211 INFO [2018-09-07 12:23:47,076] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,076] breeze.optimize.LBFGS: Val and Grad Norm: 0.0305388 (rel: 0.374) 0.0289814 INFO [2018-09-07 12:23:47,110] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,110] breeze.optimize.LBFGS: Val and Grad Norm: 0.0142904 (rel: 0.532) 0.0152334 INFO [2018-09-07 12:23:47,142] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,142] breeze.optimize.LBFGS: Val and Grad Norm: 0.00615485 (rel: 0.569) 0.0151453 INFO [2018-09-07 12:23:47,181] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,182] breeze.optimize.LBFGS: Val and Grad Norm: 0.00445325 (rel: 0.276) 0.00885304 INFO [2018-09-07 12:23:47,209] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,209] breeze.optimize.LBFGS: Val and Grad Norm: 0.00358444 (rel: 0.195) 0.00453629 INFO [2018-09-07 12:23:47,245] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,245] breeze.optimize.LBFGS: Val and Grad Norm: 0.00315597 (rel: 0.120) 0.00263045 INFO [2018-09-07 12:23:47,271] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,271] breeze.optimize.LBFGS: Val and Grad Norm: 0.00273808 (rel: 0.132) 0.00189031 INFO [2018-09-07 12:23:47,302] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,302] breeze.optimize.LBFGS: Val and Grad Norm: 0.00198476 (rel: 0.275) 0.00396367 INFO [2018-09-07 12:23:47,332] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,332] breeze.optimize.LBFGS: Val and Grad Norm: 0.00176903 (rel: 0.109) 0.00240285 INFO [2018-09-07 12:23:47,358] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,359] breeze.optimize.LBFGS: Val and Grad Norm: 0.00171634 (rel: 0.0298) 0.00378859 INFO [2018-09-07 12:23:47,390] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,390] breeze.optimize.LBFGS: Val and Grad Norm: 0.00149460 (rel: 0.129) 0.00303867 INFO [2018-09-07 12:23:47,417] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,417] breeze.optimize.LBFGS: Val and Grad Norm: 0.00144325 (rel: 0.0344) 0.00125699 INFO [2018-09-07 12:23:47,447] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,448] breeze.optimize.LBFGS: Val and Grad Norm: 0.00141001 (rel: 0.0230) 0.00131555 INFO [2018-09-07 12:23:47,481] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,481] breeze.optimize.LBFGS: Val and Grad Norm: 0.00130255 (rel: 0.0762) 0.00234124 INFO [2018-09-07 12:23:47,512] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,512] breeze.optimize.LBFGS: Val and Grad Norm: 0.00111835 (rel: 0.141) 0.00146473 INFO [2018-09-07 12:23:47,539] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,540] breeze.optimize.LBFGS: Val and Grad Norm: 0.00101016 (rel: 0.0967) 0.000644611 INFO [2018-09-07 12:23:47,567] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,567] breeze.optimize.LBFGS: Val and Grad Norm: 0.000896423 (rel: 0.113) 0.000572319 INFO [2018-09-07 12:23:47,600] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,600] breeze.optimize.LBFGS: Val and Grad Norm: 0.000764680 (rel: 0.147) 0.00102649 INFO [2018-09-07 12:23:47,628] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,629] breeze.optimize.LBFGS: Val and Grad Norm: 0.000569044 (rel: 0.256) 0.00113741 INFO [2018-09-07 12:23:47,656] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,656] breeze.optimize.LBFGS: Val and Grad Norm: 0.000477298 (rel: 0.161) 0.00295124 INFO [2018-09-07 12:23:47,681] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,681] breeze.optimize.LBFGS: Val and Grad Norm: 0.000393523 (rel: 0.176) 0.000456236 INFO [2018-09-07 12:23:47,709] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,710] breeze.optimize.LBFGS: Val and Grad Norm: 0.000353670 (rel: 0.101) 0.000377091 INFO [2018-09-07 12:23:47,770] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.41395597216906255 fval: 2.939880440891169E-4 rhs: 3.5365605592960404E-4 cdd: 1.4555228238357108E-4 INFO [2018-09-07 12:23:47,770] breeze.optimize.LBFGS: Step Size: 0.4140 INFO [2018-09-07 12:23:47,771] breeze.optimize.LBFGS: Val and Grad Norm: 0.000293988 (rel: 0.169) 0.00247088 INFO [2018-09-07 12:23:47,798] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,799] breeze.optimize.LBFGS: Val and Grad Norm: 0.000260983 (rel: 0.112) 0.000349010 INFO [2018-09-07 12:23:47,834] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,835] breeze.optimize.LBFGS: Val and Grad Norm: 0.000256205 (rel: 0.0183) 0.000165861 INFO [2018-09-07 12:23:47,870] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,870] breeze.optimize.LBFGS: Val and Grad Norm: 0.000246415 (rel: 0.0382) 0.000158839 INFO [2018-09-07 12:23:47,914] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,914] breeze.optimize.LBFGS: Val and Grad Norm: 0.000220971 (rel: 0.103) 0.000310238 INFO [2018-09-07 12:23:47,961] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,961] breeze.optimize.LBFGS: Val and Grad Norm: 0.000199341 (rel: 0.0979) 0.000316531 INFO [2018-09-07 12:23:47,990] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:47,991] breeze.optimize.LBFGS: Val and Grad Norm: 0.000192122 (rel: 0.0362) 0.000244168 INFO [2018-09-07 12:23:48,027] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:48,028] breeze.optimize.LBFGS: Val and Grad Norm: 0.000185820 (rel: 0.0328) 0.000144596 INFO [2018-09-07 12:23:48,054] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:48,055] breeze.optimize.LBFGS: Val and Grad Norm: 0.000178975 (rel: 0.0368) 8.73248e-05 INFO [2018-09-07 12:23:48,082] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:48,083] breeze.optimize.LBFGS: Val and Grad Norm: 0.000167612 (rel: 0.0635) 0.000254649 INFO [2018-09-07 12:23:48,139] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.5343477077449379 fval: 1.6324764864053553E-4 rhs: 1.6761162500423415E-4 cdd: -1.808188268014426E-7 INFO [2018-09-07 12:23:48,139] breeze.optimize.LBFGS: Step Size: 0.5343 INFO [2018-09-07 12:23:48,140] breeze.optimize.LBFGS: Val and Grad Norm: 0.000163248 (rel: 0.0260) 0.000175357 INFO [2018-09-07 12:23:48,169] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:48,169] breeze.optimize.LBFGS: Val and Grad Norm: 0.000154807 (rel: 0.0517) 0.000128822 INFO [2018-09-07 12:23:48,198] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:48,198] breeze.optimize.LBFGS: Val and Grad Norm: 0.000146059 (rel: 0.0565) 0.000285823 INFO [2018-09-07 12:23:48,225] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:48,225] breeze.optimize.LBFGS: Val and Grad Norm: 0.000141728 (rel: 0.0297) 0.000306656 INFO [2018-09-07 12:23:48,253] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:48,253] breeze.optimize.LBFGS: Val and Grad Norm: 0.000140354 (rel: 0.00969) 4.08872e-05 INFO [2018-09-07 12:23:48,280] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:48,280] breeze.optimize.LBFGS: Val and Grad Norm: 0.000139914 (rel: 0.00313) 3.77367e-05 INFO [2018-09-07 12:23:48,341] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.32733432148416675 fval: 1.3859266334177045E-4 rhs: 1.399138366825684E-4 cdd: 2.8547567605787048E-8 INFO [2018-09-07 12:23:48,341] breeze.optimize.LBFGS: Step Size: 0.3273 INFO [2018-09-07 12:23:48,341] breeze.optimize.LBFGS: Val and Grad Norm: 0.000138593 (rel: 0.00944) 0.000170698 INFO [2018-09-07 12:23:48,366] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:48,366] breeze.optimize.LBFGS: Val and Grad Norm: 0.000137064 (rel: 0.0110) 0.000108309 INFO [2018-09-07 12:23:48,392] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:48,393] breeze.optimize.LBFGS: Val and Grad Norm: 0.000129206 (rel: 0.0573) 5.59448e-05 INFO [2018-09-07 12:23:48,445] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.5023264724596983 fval: 1.264335812830162E-4 rhs: 1.292059514106151E-4 cdd: -2.7357982886147433E-8 INFO [2018-09-07 12:23:48,445] breeze.optimize.LBFGS: Step Size: 0.5023 INFO [2018-09-07 12:23:48,445] breeze.optimize.LBFGS: Val and Grad Norm: 0.000126434 (rel: 0.0215) 0.000441541 INFO [2018-09-07 12:23:48,471] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:48,471] breeze.optimize.LBFGS: Val and Grad Norm: 0.000124224 (rel: 0.0175) 0.000300558 INFO [2018-09-07 12:23:48,472] com.thingworx.analytics.training.Learner: Finished training Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]] after: {} INFO [2018-09-07 12:23:48,472] com.thingworx.analytics.training.Learner: Finished training Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0] after: {} INFO [2018-09-07 12:23:48,472] com.thingworx.analytics.training.Learner: Finished training Learner [trainer=MultiGoalTrainer [Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0]], transformer=DoNothingTransformer []] after: {} INFO [2018-09-07 12:23:48,699] com.thingworx.analytics.training.Learner: Begin training Learner [trainer=MultiGoalTrainer [Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0]], transformer=DoNothingTransformer []] INFO [2018-09-07 12:23:48,703] com.thingworx.analytics.training.Learner: Begin training Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0] INFO [2018-09-07 12:23:48,705] com.thingworx.analytics.training.ensemble.AbstractEnsembleModel: Training ensemble SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]] INFO [2018-09-07 12:23:48,714] com.thingworx.analytics.training.Learner: Begin training Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]] INFO [2018-09-07 12:23:48,719] com.thingworx.analytics.signals.SignalAnalyzer: Getting signals 1 at a time. WARN [2018-09-07 12:23:49,073] org.apache.spark.executor.Executor: 1 block locks were not released by TID = 578: [rdd_280_0] WARN [2018-09-07 12:23:49,119] org.apache.spark.executor.Executor: 1 block locks were not released by TID = 579: [rdd_280_0] INFO [2018-09-07 12:23:49,391] breeze.optimize.StrongWolfeLineSearch: Line search t: 11.173116132517862 fval: 0.07744086342465924 rhs: 0.08555391150982179 cdd: -8.609948004925237E-6 INFO [2018-09-07 12:23:49,391] breeze.optimize.LBFGS: Step Size: 11.17 INFO [2018-09-07 12:23:49,391] breeze.optimize.LBFGS: Val and Grad Norm: 0.0774409 (rel: 0.0948) 0.0351046 INFO [2018-09-07 12:23:49,465] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:49,466] breeze.optimize.LBFGS: Val and Grad Norm: 0.0705805 (rel: 0.0886) 0.0339955 INFO [2018-09-07 12:23:49,532] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:49,533] breeze.optimize.LBFGS: Val and Grad Norm: 0.0356841 (rel: 0.494) 0.0286711 INFO [2018-09-07 12:23:49,594] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:49,595] breeze.optimize.LBFGS: Val and Grad Norm: 0.0218717 (rel: 0.387) 0.0477232 INFO [2018-09-07 12:23:49,666] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:49,666] breeze.optimize.LBFGS: Val and Grad Norm: 0.0127677 (rel: 0.416) 0.0248049 INFO [2018-09-07 12:23:49,731] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:49,731] breeze.optimize.LBFGS: Val and Grad Norm: 0.00741119 (rel: 0.420) 0.0114372 INFO [2018-09-07 12:23:49,797] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:49,797] breeze.optimize.LBFGS: Val and Grad Norm: 0.00558259 (rel: 0.247) 0.00641309 INFO [2018-09-07 12:23:49,867] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:49,867] breeze.optimize.LBFGS: Val and Grad Norm: 0.00450612 (rel: 0.193) 0.00385934 INFO [2018-09-07 12:23:49,931] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:49,931] breeze.optimize.LBFGS: Val and Grad Norm: 0.00371528 (rel: 0.176) 0.00219380 INFO [2018-09-07 12:23:49,996] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:49,996] breeze.optimize.LBFGS: Val and Grad Norm: 0.00329898 (rel: 0.112) 0.00249306 INFO [2018-09-07 12:23:50,067] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:50,067] breeze.optimize.LBFGS: Val and Grad Norm: 0.00313092 (rel: 0.0509) 0.00204638 INFO [2018-09-07 12:23:50,140] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:50,140] breeze.optimize.LBFGS: Val and Grad Norm: 0.00274863 (rel: 0.122) 0.00146976 INFO [2018-09-07 12:23:50,201] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:50,201] breeze.optimize.LBFGS: Val and Grad Norm: 0.00218671 (rel: 0.204) 0.00219283 INFO [2018-09-07 12:23:50,262] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:50,262] breeze.optimize.LBFGS: Val and Grad Norm: 0.00166174 (rel: 0.240) 0.00117061 INFO [2018-09-07 12:23:50,326] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:50,326] breeze.optimize.LBFGS: Val and Grad Norm: 0.00151600 (rel: 0.0877) 0.00615004 INFO [2018-09-07 12:23:50,391] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:50,391] breeze.optimize.LBFGS: Val and Grad Norm: 0.00116185 (rel: 0.234) 0.000306235 INFO [2018-09-07 12:23:50,470] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:50,470] breeze.optimize.LBFGS: Val and Grad Norm: 0.00113889 (rel: 0.0198) 0.000208230 INFO [2018-09-07 12:23:50,556] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:50,556] breeze.optimize.LBFGS: Val and Grad Norm: 0.00111586 (rel: 0.0202) 0.000303862 INFO [2018-09-07 12:23:50,617] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:50,617] breeze.optimize.LBFGS: Val and Grad Norm: 0.00108507 (rel: 0.0276) 0.000349261 INFO [2018-09-07 12:23:50,752] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.1 fval: 0.0010311552576588966 rhs: 0.0010850663671015716 cdd: -4.965555075124332E-4 INFO [2018-09-07 12:23:50,815] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.19 fval: 0.0010082197929866077 rhs: 0.0010850615345048891 cdd: 1.6342998579109755E-4 INFO [2018-09-07 12:23:50,815] breeze.optimize.LBFGS: Step Size: 0.1900 INFO [2018-09-07 12:23:50,815] breeze.optimize.LBFGS: Val and Grad Norm: 0.00100822 (rel: 0.0708) 0.00290335 INFO [2018-09-07 12:23:50,879] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:50,879] breeze.optimize.LBFGS: Val and Grad Norm: 0.000923889 (rel: 0.0836) 0.00281859 INFO [2018-09-07 12:23:51,008] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.509101001242868 fval: 6.890820941408608E-4 rhs: 9.238437582780391E-4 cdd: 5.4538042409164205E-6 INFO [2018-09-07 12:23:51,008] breeze.optimize.LBFGS: Step Size: 0.5091 INFO [2018-09-07 12:23:51,008] breeze.optimize.LBFGS: Val and Grad Norm: 0.000689082 (rel: 0.254) 0.00140160 INFO [2018-09-07 12:23:51,131] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.3927175964657892 fval: 6.755033525825601E-4 rhs: 6.890792410036756E-4 cdd: 2.6945051806051415E-7 INFO [2018-09-07 12:23:51,131] breeze.optimize.LBFGS: Step Size: 0.3927 INFO [2018-09-07 12:23:51,131] breeze.optimize.LBFGS: Val and Grad Norm: 0.000675503 (rel: 0.0197) 0.000328185 INFO [2018-09-07 12:23:51,197] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:51,197] breeze.optimize.LBFGS: Val and Grad Norm: 0.000646192 (rel: 0.0434) 0.00181094 INFO [2018-09-07 12:23:51,258] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:51,258] breeze.optimize.LBFGS: Val and Grad Norm: 0.000616482 (rel: 0.0460) 0.00153005 INFO [2018-09-07 12:23:51,322] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:51,323] breeze.optimize.LBFGS: Val and Grad Norm: 0.000516457 (rel: 0.162) 0.00282725 INFO [2018-09-07 12:23:51,384] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:51,384] breeze.optimize.LBFGS: Val and Grad Norm: 0.000443381 (rel: 0.141) 0.000389069 INFO [2018-09-07 12:23:51,446] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:51,447] breeze.optimize.LBFGS: Val and Grad Norm: 0.000421189 (rel: 0.0501) 0.000793768 INFO [2018-09-07 12:23:51,518] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:51,518] breeze.optimize.LBFGS: Val and Grad Norm: 0.000408034 (rel: 0.0312) 0.000164779 INFO [2018-09-07 12:23:51,582] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:51,582] breeze.optimize.LBFGS: Val and Grad Norm: 0.000400764 (rel: 0.0178) 0.000168707 INFO [2018-09-07 12:23:51,645] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:51,645] breeze.optimize.LBFGS: Val and Grad Norm: 0.000395788 (rel: 0.0124) 0.000216850 INFO [2018-09-07 12:23:51,707] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:51,707] breeze.optimize.LBFGS: Val and Grad Norm: 0.000393315 (rel: 0.00625) 0.000252750 INFO [2018-09-07 12:23:51,781] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:51,781] breeze.optimize.LBFGS: Val and Grad Norm: 0.000388750 (rel: 0.0116) 0.000230467 INFO [2018-09-07 12:23:51,847] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:51,847] breeze.optimize.LBFGS: Val and Grad Norm: 0.000370925 (rel: 0.0459) 0.000214087 INFO [2018-09-07 12:23:51,908] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:51,909] breeze.optimize.LBFGS: Val and Grad Norm: 0.000354620 (rel: 0.0440) 0.000987145 INFO [2018-09-07 12:23:51,969] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:51,970] breeze.optimize.LBFGS: Val and Grad Norm: 0.000340639 (rel: 0.0394) 0.00106475 INFO [2018-09-07 12:23:52,031] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:52,031] breeze.optimize.LBFGS: Val and Grad Norm: 0.000339293 (rel: 0.00395) 0.00103635 INFO [2018-09-07 12:23:52,094] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:52,094] breeze.optimize.LBFGS: Val and Grad Norm: 0.000320260 (rel: 0.0561) 0.000489900 INFO [2018-09-07 12:23:52,217] breeze.optimize.StrongWolfeLineSearch: Line search t: 0.21170500372888912 fval: 3.162424741450806E-4 rhs: 3.2025901819739475E-4 cdd: -6.996991502854678E-7 INFO [2018-09-07 12:23:52,217] breeze.optimize.LBFGS: Step Size: 0.2117 INFO [2018-09-07 12:23:52,217] breeze.optimize.LBFGS: Val and Grad Norm: 0.000316242 (rel: 0.0125) 0.000121124 INFO [2018-09-07 12:23:52,278] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:52,279] breeze.optimize.LBFGS: Val and Grad Norm: 0.000313970 (rel: 0.00719) 8.99233e-05 INFO [2018-09-07 12:23:52,340] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:52,340] breeze.optimize.LBFGS: Val and Grad Norm: 0.000312706 (rel: 0.00402) 9.37946e-05 INFO [2018-09-07 12:23:52,401] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:52,401] breeze.optimize.LBFGS: Val and Grad Norm: 0.000310702 (rel: 0.00641) 5.67253e-05 INFO [2018-09-07 12:23:52,462] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:52,462] breeze.optimize.LBFGS: Val and Grad Norm: 0.000305503 (rel: 0.0167) 0.000151635 INFO [2018-09-07 12:23:52,523] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:52,523] breeze.optimize.LBFGS: Val and Grad Norm: 0.000302623 (rel: 0.00943) 0.000104520 INFO [2018-09-07 12:23:52,586] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:52,587] breeze.optimize.LBFGS: Val and Grad Norm: 0.000299015 (rel: 0.0119) 0.000114000 INFO [2018-09-07 12:23:52,649] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:52,649] breeze.optimize.LBFGS: Val and Grad Norm: 0.000297105 (rel: 0.00639) 8.91940e-05 INFO [2018-09-07 12:23:52,712] breeze.optimize.LBFGS: Step Size: 1.000 INFO [2018-09-07 12:23:52,712] breeze.optimize.LBFGS: Val and Grad Norm: 0.000295123 (rel: 0.00667) 0.000106699 INFO [2018-09-07 12:23:52,712] com.thingworx.analytics.training.Learner: Finished training Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]] after: {} INFO [2018-09-07 12:23:52,714] com.thingworx.analytics.training.Learner: Finished training Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0] after: {} INFO [2018-09-07 12:23:52,714] com.thingworx.analytics.training.Learner: Finished training Learner [trainer=MultiGoalTrainer [Learner [trainer=SoloistModel []All learners [learners=[Learner [trainer=com.thingworx.analytics.training.neuralnet.SparkAnnTrainer@47501a7b, transformer=NeuralNetTransformerFactory [maxNumberOfMiningFields=25 expanding=true]]]], transformer=com.thingworx.analytics.training.xforms.NormalizeTransformerFactory@2f7f5ea0]], transformer=DoNothingTransformer []] after: {} ERROR [2018-09-07 12:23:54,455] com.thingworx.analytics.ccc.job.local.AsynchronousMemoryJobExecutor: Call to TrainingJob [20180907] failed due to: ! com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException: Unrecognized field "code" (class com.thingworx.analytics.results.api.ResultResponse), not marked as ignorable (7 known properties: "status", "tags", "id", "description", "lastModified", "name", "resultUri"]) ! at [Source: org.glassfish.jersey.message.internal.ReaderInterceptorExecutor$UnCloseableInputStream@56219ffd; line: 1, column: 12] (through reference chain: com.thingworx.analytics.results.api.ResultResponse["code"]) ! at com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException.from(UnrecognizedPropertyException.java:62) ! at com.fasterxml.jackson.databind.DeserializationContext.reportUnknownProperty(DeserializationContext.java:851) ! at com.fasterxml.jackson.databind.deser.std.StdDeserializer.handleUnknownProperty(StdDeserializer.java:1085) ! at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownProperty(BeanDeserializerBase.java:1388) ! at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.handleUnknownVanilla(BeanDeserializerBase.java:1366) ! at com.fasterxml.jackson.databind.deser.BeanDeserializer.vanillaDeserialize(BeanDeserializer.java:266) ! at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:125) ! at com.fasterxml.jackson.databind.ObjectReader._bind(ObjectReader.java:1536) ! at com.fasterxml.jackson.databind.ObjectReader.readValue(ObjectReader.java:944) ! at com.fasterxml.jackson.jaxrs.base.ProviderBase.readFrom(ProviderBase.java:811) ! at org.glassfish.jersey.message.internal.ReaderInterceptorExecutor$TerminalReaderInterceptor.invokeReadFrom(ReaderInterceptorExecutor.java:256) ! at org.glassfish.jersey.message.internal.ReaderInterceptorExecutor$TerminalReaderInterceptor.aroundReadFrom(ReaderInterceptorExecutor.java:235) ! at org.glassfish.jersey.message.internal.ReaderInterceptorExecutor.proceed(ReaderInterceptorExecutor.java:155) ! at org.glassfish.jersey.message.internal.MessageBodyFactory.readFrom(MessageBodyFactory.java:1085) ! at org.glassfish.jersey.message.internal.InboundMessageContext.readEntity(InboundMessageContext.java:874) ! ... 24 common frames omitted ! Causing: javax.ws.rs.ProcessingException: Error reading entity from input stream. ! at org.glassfish.jersey.message.internal.InboundMessageContext.readEntity(InboundMessageContext.java:889) ! at org.glassfish.jersey.message.internal.InboundMessageContext.readEntity(InboundMessageContext.java:808) ! at org.glassfish.jersey.client.ClientResponse.readEntity(ClientResponse.java:326) ! at org.glassfish.jersey.client.InboundJaxrsResponse$1.call(InboundJaxrsResponse.java:115) ! at org.glassfish.jersey.internal.Errors.process(Errors.java:315) ! at org.glassfish.jersey.internal.Errors.process(Errors.java:297) ! at org.glassfish.jersey.internal.Errors.process(Errors.java:228) ! at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:419) ! at org.glassfish.jersey.client.InboundJaxrsResponse.runInScopeIfPossible(InboundJaxrsResponse.java:267) ! at org.glassfish.jersey.client.InboundJaxrsResponse.readEntity(InboundJaxrsResponse.java:112) ! at com.thingworx.analytics.training.results.model.impl.ModelServiceModelSource.store(ModelServiceModelSource.java:131) ! at com.thingworx.analytics.training.results.model.impl.ModelServiceModelSource.store(ModelServiceModelSource.java:46) ! at com.thingworx.analytics.training.core.TrainingRunner.storeModel(TrainingRunner.java:166) ! at com.thingworx.analytics.training.core.TrainingRunner.run(TrainingRunner.java:127) ! at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ! at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ! at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ! at java.lang.reflect.Method.invoke(Method.java:498) ! at com.thingworx.analytics.training.core.TrainingJob.run(TrainingJob.java:72) ! at com.thingworx.analytics.ccc.job.local.AsynchronousMemoryJobExecutor$1.call(AsynchronousMemoryJobExecutor.java:88) ! at com.thingworx.analytics.ccc.job.local.AsynchronousMemoryJobExecutor$1.call(AsynchronousMemoryJobExecutor.java:77) ! at java.util.concurrent.FutureTask.run(FutureTask.java:266) ! at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ! at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ! at java.lang.Thread.run(Thread.java:748)