Merge pull request #10671 from jakubikan/category-for-market-cap-pairlist

Category for market cap pairlist

docs/includes/pairlists.md

@@ -360,14 +360,21 @@ The optional `bearer_token` will be included in the requests Authorization Heade
         "method": "MarketCapPairList",
         "number_assets": 20,
         "max_rank": 50,
-        "refresh_period": 86400
+        "refresh_period": 86400,
+        "categories": ["layer-1"]
     }
 ]
 ```
 
 `number_assets` defines the maximum number of pairs returned by the pairlist. `max_rank` will determine the maximum rank used in creating/filtering the pairlist. It's expected that some coins within the top `max_rank` marketcap will not be included in the resulting pairlist since not all pairs will have active trading pairs in your preferred market/stake/exchange combination.
 
-`refresh_period` setting defines the period (in seconds) at which the marketcap rank data will be refreshed. Defaults to 86,400s (1 day). The pairlist cache (`refresh_period`) is applicable on both generating pairlists (first position in the list) and filtering instances (not the first position in the list).
+The `refresh_period` setting defines the interval (in seconds) at which the marketcap rank data will be refreshed. The default is 86,400 seconds (1 day). The pairlist cache (`refresh_period`) applies to both generating pairlists (when in the first position in the list) and filtering instances (when not in the first position in the list).
+
+The `categories` setting specifies the [coingecko categories](https://www.coingecko.com/en/categories) from which to select coins from. The default is an empty list `[]`, meaning no category filtering is applied.
+If an incorrect category string is chosen, the plugin will print the available categories from CoinGecko and fail. The category should be the ID of the category, for example, for `https://www.coingecko.com/en/categories/layer-1`, the category ID would be `layer-1`. You can pass multiple categories such as `["layer-1", "meme-token"]` to select from several categories.
+
+!!! Warning "Many categories"
+    Each added category corresponds to one API call to CoinGecko. The more categories you add, the longer the pairlist generation will take, potentially causing rate limit issues.
 
 #### AgeFilter
 

freqtrade/plugins/pairlist/IPairList.py

@@ -39,6 +39,11 @@ class __OptionPairlistParameter(__PairlistParameterBase):
     options: List[str]
 
 
+class __ListPairListParamenter(__PairlistParameterBase):
+    type: Literal["list"]
+    default: Union[List[str], None]
+
+
 class __BoolPairlistParameter(__PairlistParameterBase):
     type: Literal["boolean"]
     default: Union[bool, None]
@@ -49,6 +54,7 @@ PairlistParameter = Union[
     __StringPairlistParameter,
     __OptionPairlistParameter,
     __BoolPairlistParameter,
+    __ListPairListParamenter,
 ]
 
 

freqtrade/plugins/pairlist/MarketCapPairList.py

@@ -35,6 +35,7 @@ class MarketCapPairList(IPairList):
         self._number_assets = self._pairlistconfig["number_assets"]
         self._max_rank = self._pairlistconfig.get("max_rank", 30)
         self._refresh_period = self._pairlistconfig.get("refresh_period", 86400)
+        self._categories = self._pairlistconfig.get("categories", [])
         self._marketcap_cache: TTLCache = TTLCache(maxsize=1, ttl=self._refresh_period)
         self._def_candletype = self._config["candle_type_def"]
 
@@ -45,6 +46,17 @@ class MarketCapPairList(IPairList):
             is_demo=_coingecko_config.get("is_demo", True),
         )
 
+        if self._categories:
+            categories = self._coingecko.get_coins_categories_list()
+            category_ids = [cat["category_id"] for cat in categories]
+
+            for category in self._categories:
+                if category not in category_ids:
+                    raise OperationalException(
+                        f"Category {category} not in coingecko category list. "
+                        f"You can choose from {category_ids}"
+                    )
+
         if self._max_rank > 250:
             raise OperationalException("This filter only support marketcap rank up to 250.")
 
@@ -85,6 +97,15 @@ class MarketCapPairList(IPairList):
                 "description": "Max rank of assets",
                 "help": "Maximum rank of assets to use from the pairlist",
             },
+            "categories": {
+                "type": "list",
+                "default": [],
+                "description": "Coin Categories",
+                "help": (
+                    "The Category of the coin e.g layer-1 default [] "
+                    "(https://www.coingecko.com/en/categories)"
+                ),
+            },
             "refresh_period": {
                 "type": "number",
                 "default": 86400,
@@ -132,15 +153,29 @@ class MarketCapPairList(IPairList):
         """
         marketcap_list = self._marketcap_cache.get("marketcap")
 
+        default_kwargs = {
+            "vs_currency": "usd",
+            "order": "market_cap_desc",
+            "per_page": "250",
+            "page": "1",
+            "sparkline": "false",
+            "locale": "en",
+        }
+
         if marketcap_list is None:
-            data = self._coingecko.get_coins_markets(
+            data = []
-                vs_currency="usd",
+
-                order="market_cap_desc",
+            if not self._categories:
-                per_page="250",
+                data = self._coingecko.get_coins_markets(**default_kwargs)
-                page="1",
+            else:
-                sparkline="false",
+                for category in self._categories:
-                locale="en",
+                    category_data = self._coingecko.get_coins_markets(
-            )
+                        **default_kwargs, **({"category": category} if category else {})
+                    )
+                    data += category_data
+
+            data.sort(key=lambda d: float(d.get("market_cap") or 0.0), reverse=True)
+
             if data:
                 marketcap_list = [row["symbol"] for row in data]
                 self._marketcap_cache["marketcap"] = marketcap_list
@@ -157,7 +192,7 @@ class MarketCapPairList(IPairList):
 
             for mc_pair in top_marketcap:
                 test_pair = f"{mc_pair.upper()}/{pair_format}"
-                if test_pair in pairlist:
+                if test_pair in pairlist and test_pair not in filtered_pairlist:
                     filtered_pairlist.append(test_pair)
                     if len(filtered_pairlist) == self._number_assets:
                         break

tests/plugins/test_pairlist.py

@@ -2212,7 +2212,7 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None:
 
 
 @pytest.mark.parametrize(
-    "pairlists,trade_mode,result",
+    "pairlists,trade_mode,result,coin_market_calls",
     [
         (
             [
@@ -2222,6 +2222,7 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None:
             ],
             "spot",
             ["BTC/USDT", "ETH/USDT"],
+            1,
         ),
         (
             [
@@ -2231,6 +2232,7 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None:
             ],
             "spot",
             ["BTC/USDT", "ETH/USDT", "XRP/USDT", "ADA/USDT"],
+            1,
         ),
         (
             [
@@ -2240,6 +2242,7 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None:
             ],
             "spot",
             ["BTC/USDT", "ETH/USDT", "XRP/USDT"],
+            1,
         ),
         (
             [
@@ -2249,6 +2252,7 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None:
             ],
             "spot",
             ["BTC/USDT", "ETH/USDT", "XRP/USDT"],
+            1,
         ),
         (
             [
@@ -2257,6 +2261,7 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None:
             ],
             "spot",
             ["BTC/USDT", "ETH/USDT", "XRP/USDT"],
+            1,
         ),
         (
             [
@@ -2265,6 +2270,7 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None:
             ],
             "spot",
             ["BTC/USDT", "ETH/USDT"],
+            1,
         ),
         (
             [
@@ -2273,6 +2279,7 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None:
             ],
             "futures",
             ["ETH/USDT:USDT"],
+            1,
         ),
         (
             [
@@ -2281,11 +2288,34 @@ def test_FullTradesFilter(mocker, default_conf_usdt, fee, caplog) -> None:
             ],
             "futures",
             ["ETH/USDT:USDT", "ADA/USDT:USDT"],
+            1,
+        ),
+        (
+            [
+                # MarketCapPairList as generator - futures, 1 category
+                {"method": "MarketCapPairList", "number_assets": 2, "categories": ["layer-1"]}
+            ],
+            "futures",
+            ["ETH/USDT:USDT", "ADA/USDT:USDT"],
+            ["layer-1"],
+        ),
+        (
+            [
+                # MarketCapPairList as generator - futures, 1 category
+                {
+                    "method": "MarketCapPairList",
+                    "number_assets": 2,
+                    "categories": ["layer-1", "protocol"],
+                }
+            ],
+            "futures",
+            ["ETH/USDT:USDT", "ADA/USDT:USDT"],
+            ["layer-1", "protocol"],
         ),
     ],
 )
 def test_MarketCapPairList_filter(
-    mocker, default_conf_usdt, trade_mode, markets, pairlists, result
+    mocker, default_conf_usdt, trade_mode, markets, pairlists, result, coin_market_calls
 ):
     test_value = [
         {"symbol": "btc"},
@@ -2309,8 +2339,16 @@ def test_MarketCapPairList_filter(
         markets=PropertyMock(return_value=markets),
         exchange_has=MagicMock(return_value=True),
     )
-
     mocker.patch(
+        "freqtrade.plugins.pairlist.MarketCapPairList.FtCoinGeckoApi.get_coins_categories_list",
+        return_value=[
+            {"category_id": "layer-1"},
+            {"category_id": "protocol"},
+            {"category_id": "defi"},
+        ],
+    )
+
+    gcm_mock = mocker.patch(
         "freqtrade.plugins.pairlist.MarketCapPairList.FtCoinGeckoApi.get_coins_markets",
         return_value=test_value,
     )
@@ -2319,6 +2357,15 @@ def test_MarketCapPairList_filter(
 
     pm = PairListManager(exchange, default_conf_usdt)
     pm.refresh_pairlist()
+    if isinstance(coin_market_calls, int):
+        assert gcm_mock.call_count == coin_market_calls
+    else:
+        assert gcm_mock.call_count == len(coin_market_calls)
+        for call in coin_market_calls:
+            assert any(
+                "category" in c.kwargs and c.kwargs["category"] == call
+                for c in gcm_mock.call_args_list
+            )
 
     assert pm.whitelist == result
 
@@ -2391,6 +2438,27 @@ def test_MarketCapPairList_exceptions(mocker, default_conf_usdt):
     ):
         PairListManager(exchange, default_conf_usdt)
 
+    # Test invalid coinmarkets list
+    mocker.patch(
+        "freqtrade.plugins.pairlist.MarketCapPairList.FtCoinGeckoApi.get_coins_categories_list",
+        return_value=[
+            {"category_id": "layer-1"},
+            {"category_id": "protocol"},
+            {"category_id": "defi"},
+        ],
+    )
+    default_conf_usdt["pairlists"] = [
+        {
+            "method": "MarketCapPairList",
+            "number_assets": 20,
+            "categories": ["layer-1", "defi", "layer250"],
+        }
+    ]
+    with pytest.raises(
+        OperationalException, match="Category layer250 not in coingecko category list."
+    ):
+        PairListManager(exchange, default_conf_usdt)
+
 
 @pytest.mark.parametrize(
     "pairlists,expected_error,expected_warning",