[DSIP-38] Support upload/download plugins from maven repo

.github/workflows/api-test.yml

@@ -83,15 +83,15 @@ jobs:
           -Dmaven.test.skip=true \
           -Dmaven.javadoc.skip=true \
           -Dspotless.skip=true \
-          -Pdocker,release -Ddocker.tag=ci
+          -Pdocker,staging -Ddocker.tag=ci
       - name: Export Docker Images
         run: |
           docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
           && du -sh /tmp/standalone-image.tar
       - uses: actions/upload-artifact@v4
         name: Upload Docker Images
         with:
-          name: standalone-image
+          name: standalone-image-api-test
           path: /tmp/standalone-image.tar
           retention-days: 1
   api-test:
@@ -140,7 +140,7 @@ jobs:
       - uses: actions/download-artifact@v4
         name: Download Docker Images
         with:
-          name: standalone-image
+          name: standalone-image-api-test
           path: /tmp
       - name: Load Docker Images
         run: |

.github/workflows/backend.yml

@@ -89,7 +89,7 @@ jobs:
       - name: Build and Package on ${{ matrix.java }}
         run: |
           ./mvnw -B clean install \
-                 -Prelease \
+                 -Pstaging \
                  -Dmaven.test.skip=true \
                  -Dspotless.skip=true
       - name: Check dependency license

.github/workflows/e2e-k8s.yml

@@ -73,7 +73,7 @@ jobs:
           -Dmaven.test.skip=true \
           -Dmaven.javadoc.skip=true \
           -Dspotless.skip=true \
-          -Pdocker,release -Ddocker.tag=ci
+          -Pdocker,staging -Ddocker.tag=ci
       - name: Create k8s Kind Cluster
         run: |
           # install kubectl

.github/workflows/e2e.yml

@@ -85,15 +85,15 @@ jobs:
           -Dmaven.test.skip=true \
           -Dmaven.javadoc.skip=true \
           -Dspotless.skip=true \
-          -Pdocker,release -Ddocker.tag=ci
+          -Pdocker,staging -Ddocker.tag=ci
       - name: Export Docker Images
         run: |
           docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \
           && du -sh /tmp/standalone-image.tar
       - uses: actions/upload-artifact@v4
         name: Upload Docker Images
         with:
-          name: standalone-image
+          name: standalone-image-e2e
           path: /tmp/standalone-image.tar
           retention-days: 1
   e2e:
@@ -168,7 +168,7 @@ jobs:
       - uses: actions/download-artifact@v4
         name: Download Docker Images
         with:
-          name: standalone-image
+          name: standalone-image-e2e
           path: /tmp
       - name: Load Docker Images
         run: |

.github/workflows/publish-nexus.yaml

@@ -0,0 +1,75 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Publish Snapshot
+
+on:
+  release:
+    types:
+      - released
+
+jobs:
+  publish-snapshot:
+    if: github.repository == 'apache/dolphinscheduler'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Maximize runner space
+        uses: ./.github/actions/maximize-build-space
+        with:
+          root-reserve-mb: 30720
+          temp-reserve-mb: 10240
+          remove-dotnet: 'true'
+          remove-android: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
+      - uses: actions/checkout@v4
+      - name: Cache local Maven repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-backend
+          restore-keys: ${{ runner.os }}-maven-
+      - name: Setup JDK 8
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'adopt'
+          java-version: 8
+      - name: Set environment variables
+        run: |
+          if [[ ${{ github.event_name }} == "schedule" ]]; then
+            echo "SETTINGS_PATH=./.github/workflows/publish-nexus/snapshot-asf-settings.xml" >> $SETTINGS_PATH
+          elif [[ ${{ github.event_name }} == "release" ]]; then
+            echo "SETTINGS_PATH=./.github/workflows/publish-nexus/release-asf-settings.xml" >> $SETTINGS_PATH
+          else
+            echo "unknown event name: ${{ github.event_name }}"
+            exit 2
+          fi
+      - name: Publish snapshot with dev branch
+        env:
+          ASF_USERNAME: ${{ secrets.NEXUS_USER }}
+          ASF_PASSWORD: ${{ secrets.NEXUS_PW }}
+        run: |
+          ./mvnw clean deploy \
+          -s ${{ env.SETTINGS_PATH }} \
+          -Dmaven.test.skip=true \
+          -Dspotless.skip=true \
+          -Pstaging

.github/workflows/publish-nexus/release-asf-settings.xml

@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd">
+    <servers>
+        <server>
+            <id>apache.snapshots.https</id>
+            <username>${env.ASF_USERNAME}</username>
+            <password>${env.ASF_PASSWORD}</password>
+        </server>
+        <server>
+            <id>apache.releases.https</id>
+            <username>${env.ASF_USERNAME}</username>
+            <password>${env.ASF_PASSWORD}</password>
+        </server>
+    </servers>
+</settings>

.github/workflows/publish-nexus/snapshot-asf-settings.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd">
+    <servers>
+        <server>
+            <id>apache.snapshots.https</id>
+            <username>${env.ASF_USERNAME}</username>
+            <password>${env.ASF_PASSWORD}</password>
+        </server>
+    </servers>
+</settings>

.licenserc.yaml

@@ -48,6 +48,7 @@ header:
     - 'org.mockito.plugins.MockMaker'
     - tools/dependencies/known-dependencies.txt
     - '**/banner.txt'
+    - '**/*-banner.txt'
     - '.terraform.lock.hcl'
     - deploy/kubernetes/dolphinscheduler/README.md.gotmpl
     - .idea/vcs.xml

config/plugins_config

@@ -0,0 +1,110 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# This mapping is used to resolve the Jar package name without version (or call artifactId)
+#
+# corresponding to the module in the user Config, helping Dolphinscheduler to load the correct Jar package.
+# Don't modify the delimiter " -- ", just select the plugin you need
+
+--alert-plugins--
+dolphinscheduler-alert-aliyunVoice
+dolphinscheduler-alert-dingtalk
+dolphinscheduler-alert-email
+dolphinscheduler-alert-feishu
+dolphinscheduler-alert-http
+dolphinscheduler-alert-pagerduty
+dolphinscheduler-alert-prometheus
+dolphinscheduler-alert-script
+dolphinscheduler-alert-slack
+dolphinscheduler-alert-telegram
+dolphinscheduler-alert-webexteams
+dolphinscheduler-alert-wechat
+--end--
+
+--datasource-plugins--
+dolphinscheduler-datasource-aliyunserverlessspark
+dolphinscheduler-datasource-athena
+dolphinscheduler-datasource-azure-sql
+dolphinscheduler-datasource-clickhouse
+dolphinscheduler-datasource-dameng
+dolphinscheduler-datasource-databend
+dolphinscheduler-datasource-db2
+dolphinscheduler-datasource-doris
+dolphinscheduler-datasource-hana
+dolphinscheduler-datasource-hive
+dolphinscheduler-datasource-k8s
+dolphinscheduler-datasource-kyuubi
+dolphinscheduler-datasource-mysql
+dolphinscheduler-datasource-oceanbase
+dolphinscheduler-datasource-oracle
+dolphinscheduler-datasource-postgresql
+dolphinscheduler-datasource-presto
+dolphinscheduler-datasource-redshift
+dolphinscheduler-datasource-sagemaker
+dolphinscheduler-datasource-snowflake
+dolphinscheduler-datasource-spark
+dolphinscheduler-datasource-sqlserver
+dolphinscheduler-datasource-ssh
+dolphinscheduler-datasource-starrocks
+dolphinscheduler-datasource-trino
+dolphinscheduler-datasource-vertica
+dolphinscheduler-datasource-zeppelin
+--end--
+
+--storage-plugins--
+dolphinscheduler-storage-abs
+dolphinscheduler-storage-gcs
+dolphinscheduler-storage-hdfs
+dolphinscheduler-storage-obs
+dolphinscheduler-storage-oss
+dolphinscheduler-storage-s3
+--end--
+
+--task-plugins--
+dolphinscheduler-task-aliyunserverlessspark
+dolphinscheduler-task-chunjun
+dolphinscheduler-task-datafactory
+dolphinscheduler-task-dataquality
+dolphinscheduler-task-datasync
+dolphinscheduler-task-datax
+dolphinscheduler-task-dinky
+dolphinscheduler-task-dms
+dolphinscheduler-task-dvc
+dolphinscheduler-task-emr
+dolphinscheduler-task-flink
+dolphinscheduler-task-flink-stream
+dolphinscheduler-task-hivecli
+dolphinscheduler-task-http
+dolphinscheduler-task-java
+dolphinscheduler-task-jupyter
+dolphinscheduler-task-k8s
+dolphinscheduler-task-kubeflow
+dolphinscheduler-task-linkis
+dolphinscheduler-task-mlflow
+dolphinscheduler-task-mr
+dolphinscheduler-task-openmldb
+dolphinscheduler-task-procedure
+dolphinscheduler-task-python
+dolphinscheduler-task-pytorch
+dolphinscheduler-task-remoteshell
+dolphinscheduler-task-sagemaker
+dolphinscheduler-task-seatunnel
+dolphinscheduler-task-shell
+dolphinscheduler-task-spark
+dolphinscheduler-task-sql
+dolphinscheduler-task-sqoop
+dolphinscheduler-task-zeppelin
+--end--

docs/docs/en/guide/installation/pseudo-cluster.md

@@ -16,10 +16,27 @@ Pseudo-cluster deployment of DolphinScheduler requires external software support
   - `pstree` for macOS
   - `psmisc` for Fedora/Red/Hat/CentOS/Ubuntu/Debian
 
-> **_Note:_** DolphinScheduler itself does not depend on Hadoop, Hive, Spark, but if you need to run tasks that depend on them, you need to have the corresponding environment support.
+## Download Plugins Dependencies
+
+Starting from version 3.3.0, the binary package no longer provides plugin dependencies, and users need to download them by themselves. The plugin dependency package download address: [Plugin Dependency Package](https://repo.maven.apache.org/maven2/org/apache/dolphinscheduler)
+You can also execute the following command to install plugin dependencies:
+
+```shell
+bash ./bin/install-plugins.sh 3.3.0
+```
+
+Usually, you do not need all connector plugins, you can specify the plugins you need by configuring `conf/plugins_config`. For example, if you only need the `dolphinscheduler-task-shell` plugin, you can modify the configuration file as follows:
+
+```
+--task-plugins--
+dolphinscheduler-task-shell
+--end--
+```
 
 ## DolphinScheduler Startup Environment
 
+> **_Note:_** DolphinScheduler itself does not depend on Hadoop, Hive, Spark, but if you need to run tasks that depend on them, you need to have the corresponding environment support.
+
 ### Configure User Exemption and Permissions
 
 Create a deployment user, and make sure to configure `sudo` without password. Here make an example to create user `dolphinscheduler`:

docs/docs/en/guide/installation/standalone.md

@@ -14,6 +14,10 @@ If you want to deploy DolphinScheduler in production, we recommend you follow [c
 - JDK：download [JDK][jdk] (1.8 or 11), install and configure environment variable `JAVA_HOME` and append `bin` dir (included in `JAVA_HOME`) to `PATH` variable. You can skip this step if it already exists in your environment.
 - Binary package: download the DolphinScheduler binary package at [download page](https://dolphinscheduler.apache.org/en-us/download/<version>).  <!-- markdown-link-check-disable-line -->
 
+## Download Plugin Dependencies
+
+Please refer to the [Download Plugin Dependencies](../installation/pseudo-cluster.md) in pseudo-cluster deployment.
+
 ### Configure User Exemption and Permissions
 
 Create a deployment user, and make sure to configure `sudo` without password. Here make an example to create user `dolphinscheduler`:

docs/docs/en/guide/upgrade/incompatible.md

@@ -28,8 +28,8 @@ This document records the incompatible updates between each version. You need to
 
 ## 3.3.0
 
-* Remove the `udf-manage` function from the `resource center` ([#16209])
-* Remove the `Pigeon` from the `Task Plugin` ([#16218])
-* Uniformly name `process` in code as `workflow` ([#16515])
-* Deprecated upgrade code of 1.x and 2.x in 3.3.0-release ([#16543])
+* Remove the `udf-manage` function from the `resource center` ([#16209])(https://github.com/apache/dolphinscheduler/pull/16209)
+* Remove the `Pigeon` from the `Task Plugin` ([#16218])(https://github.com/apache/dolphinscheduler/pull/16218)
+* Uniformly name `process` in code as `workflow` ([#16515])(https://github.com/apache/dolphinscheduler/pull/16515)
+* Deprecated upgrade code of 1.x and 2.x in 3.3.0-release ([#16543])(https://github.com/apache/dolphinscheduler/pull/16543)
 

docs/docs/en/guide/upgrade/upgrade.md

@@ -72,7 +72,8 @@ Execution result:
 
 #### Upgrade version restriction
 
-After version 3.3.X and later, we only support upgrading from 3.0.0. For versions lower than this, please download the historical version and upgrade to 3.0.0.
+- After version 3.3.X and later, we only support upgrading from 3.0.0. For versions lower than this, please download the historical version and upgrade to 3.0.0.
+- After version 3.3.X and later, binary packages no longer provide plugins dependencies by default, so when you use them for the first time, you need to download and install them yourself. For more information, please refer to [Pseudo-Cluster](../installation/pseudo-cluster.md).
 
 #### Precautions after the upgrade