Various fixes for the market + Temporal Pagerank implementation

Tribler/Test/Community/Market/Reputation/test_reputation_base.py

@@ -2,6 +2,7 @@
 
 from Tribler.Test.test_as_server import AbstractServer
 from Tribler.community.market.database import MarketDB
+from Tribler.community.market.reputation.temporal_pagerank_manager import TemporalPagerankReputationManager
 from Tribler.community.market.tradechain.block import TradeChainBlock
 
 
@@ -16,18 +17,24 @@ def setUp(self, annotate=True):
         os.mkdir(os.path.join(self.session_base_dir, 'sqlite'))
         self.market_db = MarketDB(self.session_base_dir, 'market')
 
-    def insert_transaction(self, pubkey1, pubkey2, asset1_type, asset1_amount, asset2_type, asset2_amount):
-        latest_block = self.market_db.get_latest(pubkey1)
-
-        block = TradeChainBlock()
-        block.public_key = pubkey1
-        if latest_block:
-            block.sequence_number = latest_block.sequence_number + 1
-
-        block.link_public_key = pubkey2
-
-        transaction = {"asset1_type": asset1_type, "asset1_amount": asset1_amount,
-                       "asset2_type": asset2_type, "asset2_amount": asset2_amount}
-        block.transaction = transaction
+    def insert_transaction(self, pubkey1, pubkey2, quantity, price):
+        transaction = {
+            "tx": {
+                "quantity_type": quantity.wallet_id,
+                "quantity": float(quantity),
+                "price_type": price.wallet_id,
+                "price": float(price)
+            }
+        }
+        block = TradeChainBlock.create(transaction, self.market_db, pubkey1, link=None, link_pk=pubkey2)
+        link_block = TradeChainBlock.create(transaction, self.market_db, pubkey2, link=block, link_pk=pubkey1)
 
         self.market_db.add_block(block)
+        self.market_db.add_block(link_block)
+
+    def compute_reputations(self):
+        blocks = self.market_db.get_all_blocks()
+        rep_manager = TemporalPagerankReputationManager(blocks)
+        rep = rep_manager.compute(own_public_key='a')
+        self.assertIsInstance(rep, dict)
+        return rep

Tribler/Test/Community/Market/Reputation/test_reputation_pagerank.py

@@ -1,5 +1,6 @@
 from Tribler.Test.Community.Market.Reputation.test_reputation_base import TestReputationBase
-from Tribler.community.market.reputation.pagerank_manager import PagerankReputationManager
+from Tribler.community.market.core.price import Price
+from Tribler.community.market.core.quantity import Quantity
 
 
 class TestReputationPagerank(TestReputationBase):
@@ -8,13 +9,57 @@ class TestReputationPagerank(TestReputationBase):
     """
 
     def test_pagerank_1(self):
-        self.insert_transaction('a', 'b', 1, 20, 2, 20)
-        self.insert_transaction('a', 'b', 1, 20, 2, 20)
-        self.insert_transaction('b', 'c', 1, 20, 2, 20)
-        self.insert_transaction('b', 'd', 1, 20, 2, 20)
-        self.insert_transaction('d', 'e', 1, 10, 2, 10)
+        """
+        Test a very simple Temporal Pagerank computation
+        """
+        self.insert_transaction('a', 'b', Quantity(1, 'BTC'), Price(1, 'MC'))
+        rep_dict = self.compute_reputations()
+        self.assertTrue('a' in rep_dict)
+        self.assertTrue('b' in rep_dict)
+        self.assertGreater(rep_dict['a'], 0)
+        self.assertGreater(rep_dict['b'], 0)
 
-        blocks = self.market_db.get_all_blocks()
-        rep_manager = PagerankReputationManager(blocks)
-        rep = rep_manager.compute(own_public_key='a')
-        self.assertIsInstance(rep, dict)
+    def test_pagerank_2(self):
+        """
+        Test isolated nodes during a Temporal Pagerank computation
+        """
+        self.insert_transaction('a', 'b', Quantity(1, 'BTC'), Price(1, 'MC'))
+        self.insert_transaction('c', 'd', Quantity(1, 'BTC'), Price(1, 'MC'))
+        rep_dict = self.compute_reputations()
+        self.assertTrue('c' in rep_dict)
+        self.assertTrue('d' in rep_dict)
+
+    def test_pagerank_3(self):
+        """
+        Test a more involved example of a Temporal Pagerank computation
+        """
+        self.insert_transaction('a', 'b', Quantity(1, 'BTC'), Price(1, 'MC'))
+        self.insert_transaction('b', 'c', Quantity(100, 'BTC'), Price(100, 'MC'))
+        self.insert_transaction('b', 'd', Quantity(100, 'BTC'), Price(100, 'MC'))
+        self.insert_transaction('b', 'e', Quantity(100, 'BTC'), Price(100, 'MC'))
+        rep_dict = self.compute_reputations()
+        self.assertEqual(len(rep_dict.keys()), 5)
+        for _, rep in rep_dict.iteritems():
+            self.assertGreater(rep, 0)
+
+    def test_pagerank_4(self):
+        """
+        Test an empty pagerank computation
+        """
+        rep_dict = self.compute_reputations()
+        self.assertDictEqual(rep_dict, {})
+
+    def test_pagerank_5(self):
+        """
+        Test a Temporal Pagerank computation
+        """
+        self.insert_transaction('a', 'b', Quantity(1, 'BTC'), Price(1, 'MC'))
+        self.insert_transaction('a', 'c', Quantity(1, 'BTC'), Price(2, 'MC'))
+        self.insert_transaction('a', 'd', Quantity(1, 'BTC'), Price(3, 'MC'))
+        self.insert_transaction('a', 'e', Quantity(1, 'BTC'), Price(4, 'MC'))
+        self.insert_transaction('a', 'f', Quantity(1, 'BTC'), Price(5, 'MC'))
+        self.insert_transaction('a', 'g', Quantity(1, 'BTC'), Price(6, 'MC'))
+        self.insert_transaction('a', 'h', Quantity(1, 'BTC'), Price(7, 'MC'))
+        rep_dict = self.compute_reputations()
+        self.assertTrue('c' in rep_dict)
+        self.assertTrue('d' in rep_dict)

Tribler/Test/Community/Market/test_community.py

@@ -382,9 +382,9 @@ def test_compute_reputation(self):
         """
         Test the compute_reputation method
         """
-        self.market_community.tradechain_community = MockObject()
-        self.market_community.tradechain_community.persistence = MockObject()
-        self.market_community.tradechain_community.persistence.get_all_blocks = lambda: []
+        self.market_community.persistence = MockObject()
+        self.market_community.persistence.get_all_blocks = lambda: []
+        self.market_community.persistence.close = lambda: None
         self.market_community.compute_reputation()
         self.assertFalse(self.market_community.reputation_dict)
 

Tribler/community/market/community.py

@@ -33,7 +33,7 @@
 from Tribler.community.market.payload import TradePayload, DeclinedTradePayload,\
     StartTransactionPayload, WalletInfoPayload, PaymentPayload, MatchPayload, AcceptMatchPayload, DeclineMatchPayload, \
     InfoPayload, OrderStatusRequestPayload, OrderStatusResponsePayload
-from Tribler.community.market.reputation.pagerank_manager import PagerankReputationManager
+from Tribler.community.market.reputation.temporal_pagerank_manager import TemporalPagerankReputationManager
 from Tribler.community.market.tradechain.block import TradeChainBlock
 from Tribler.community.market.wallet.tc_wallet import TrustchainWallet
 from Tribler.community.trustchain.community import TrustChainCommunity, HALF_BLOCK_BROADCAST, BLOCK_PAIR, \
@@ -93,24 +93,24 @@ class MarketCommunity(TrustChainCommunity):
 
     @classmethod
     def get_master_members(cls, dispersy):
-        # generated: Tue Mar 22 23:29:40 2016
-        # curve: NID_sect571r1
+        # generated: Thu Sep 21 09:18:42 2017
+        # curve: None
         # len: 571 bits ~ 144 bytes signature
-        # pub: 170 3081a7301006072a8648ce3d020106052b8104002703819200040159af0c0925034bba3b4ea26661828e09247236059c773
-        # dac29ac9fb84d50fa6bd8acc035127a6f5c11873915f9b9a460e116ecccccfc5db1b5d8ba86bd701886ea45d8dbbb634906989395d36
-        # 6888d008f4119ad0e7f45b9dab7fb3d78a0065c5f7a866b78cb8e59b9a7d048cc0d650c5a86bdfdabb434396d23945d1239f88de4935
-        # 467424c7cc02b6579e45f63ee
-        # pub-sha1 dda25d128ebabe6b588384d05b8ff46153f98c78
+        # pub: 170 3081a7301006072a8648ce3d020106052b81040027038192000407c78a0f74058fe70e6101709389ba198cd7f41826f160b
+        # 546e3b9ad1c634e041cbd1da82849968f9736ef1d4ccc6c7ebd41ad4c29e6988e99f0f597925dd956b1f7b809fe13fe7702a34f95662
+        # 699ad6e4756b169fd6cf61b2658818e0d0e9e144827c1b5c476603b8ba3b21a735f206e9c58a361c53f133de14b9552dfc317627e101
+        # 4c901030bca5d88cea35c7489
+        # pub-sha1 6317befc5c61e41341ed2a9590980be9ea77eb3b
         # -----BEGIN PUBLIC KEY-----
-        # MIGnMBAGByqGSM49AgEGBSuBBAAnA4GSAAQBWa8MCSUDS7o7TqJmYYKOCSRyNgWc
-        # dz2sKayfuE1Q+mvYrMA1EnpvXBGHORX5uaRg4RbszMz8XbG12LqGvXAYhupF2Nu7
-        # Y0kGmJOV02aIjQCPQRmtDn9Fudq3+z14oAZcX3qGa3jLjlm5p9BIzA1lDFqGvf2r
-        # tDQ5bSOUXRI5+I3kk1RnQkx8wCtleeRfY+4=
+        # MIGnMBAGByqGSM49AgEGBSuBBAAnA4GSAAQHx4oPdAWP5w5hAXCTiboZjNf0GCbx
+        # YLVG47mtHGNOBBy9HagoSZaPlzbvHUzMbH69Qa1MKeaYjpnw9ZeSXdlWsfe4Cf4T
+        # /ncCo0+VZiaZrW5HVrFp/Wz2GyZYgY4NDp4USCfBtcR2YDuLo7Iac18gbpxYo2HF
+        # PxM94UuVUt/DF2J+EBTJAQMLyl2IzqNcdIk=
         # -----END PUBLIC KEY-----
-        master_key = "3081a7301006072a8648ce3d020106052b81040027038192000403e6f247258f60430f570cb02f5d830426fefaec" \
-                     "76a506db6e806ea0f10ee6061996f54fe6960e19978b32a0c92ece60dc0b85deaa07b7fd13fa6e54205154f78c1a" \
-                     "294effb43801045fb17124a85e42a338275d109da989942337dbc6c3b06dc2c4c62d0c2b64f2cdfe02aad5c058be" \
-                     "23027e4b99fc7271a94d176f020543e06da7a371f9794240dae44e9bc130a1a6".decode('hex')
+        master_key = "3081a7301006072a8648ce3d020106052b81040027038192000407c78a0f74058fe70e6101709389ba198cd7f41826f" \
+                     "160b546e3b9ad1c634e041cbd1da82849968f9736ef1d4ccc6c7ebd41ad4c29e6988e99f0f597925dd956b1f7b809fe" \
+                     "13fe7702a34f95662699ad6e4756b169fd6cf61b2658818e0d0e9e144827c1b5c476603b8ba3b21a735f206e9c58a36" \
+                     "1c53f133de14b9552dfc317627e1014c901030bca5d88cea35c7489".decode('hex')
         master = dispersy.get_member(public_key=master_key)
         return [master]
 
@@ -124,7 +124,6 @@ def __init__(self, *args, **kwargs):
         self.matching_engine = None
         self.incoming_match_messages = {}  # Map of TraderId -> Message (we save all incoming matches)
         self.tribler_session = None
-        self.tradechain_community = None
         self.wallets = None
         self.transaction_manager = None
         self.reputation_dict = {}
@@ -160,7 +159,6 @@ def initialize(self, tribler_session=None, tradechain_community=None, wallets=No
 
         self.order_manager = OrderManager(order_repository)
         self.tribler_session = tribler_session
-        self.tradechain_community = tradechain_community
         self.wallets = wallets or {}
         self.transaction_manager = TransactionManager(transaction_repository)
 
@@ -916,9 +914,11 @@ def on_match_message(self, messages):
 
             if order.status != "open" or order.available_quantity == Quantity(0, order.available_quantity.wallet_id):
                 # Send a declined trade back
+                decline_reason = DeclineMatchReason.ORDER_COMPLETED if order.status != "open" \
+                    else DeclineMatchReason.OTHER
                 self.send_decline_match_message(message.payload.match_id,
                                                 message.payload.matchmaker_trader_id,
-                                                DeclineMatchReason.ORDER_COMPLETED)
+                                                decline_reason)
                 continue
 
             propose_quantity = Quantity(min(float(order.available_quantity), float(message.payload.match_quantity)),
@@ -1671,7 +1671,7 @@ def send_transaction_completed(self, transaction, block):
         del self.incoming_match_messages[transaction.match_id]
         candidate = Candidate(self.lookup_ip(match_message.payload.matchmaker_trader_id), False)
 
-        linked_block = self.market_database.get_linked(block)
+        linked_block = self.market_database.get_linked(block) or block
         self.send_block_pair(block, linked_block, candidate)
 
     def on_transaction_completed_message(self, block1, block2):
@@ -1712,5 +1712,5 @@ def compute_reputation(self):
         """
         Compute the reputation of peers in the community
         """
-        rep_manager = PagerankReputationManager(self.tradechain_community.persistence.get_all_blocks())
+        rep_manager = TemporalPagerankReputationManager(self.persistence.get_all_blocks())
         self.reputation_dict = rep_manager.compute(self.my_member.public_key)

Tribler/community/market/reputation/reputation_manager.py

@@ -1,6 +1,10 @@
+import logging
+
+
 class ReputationManager(object):
 
     def __init__(self, blocks):
+        self._logger = logging.getLogger(self.__class__.__name__)
         self.blocks = blocks
 
     def compute(self, own_public_key):

Tribler/community/market/reputation/temporal_pagerank_manager.py

@@ -0,0 +1,64 @@
+import networkx as nx
+
+from Tribler.community.market.reputation.reputation_manager import ReputationManager
+from Tribler.community.trustchain.block import UNKNOWN_SEQ
+
+
+class TemporalPagerankReputationManager(ReputationManager):
+
+    def compute(self, own_public_key):
+        """
+        Compute the reputation based on the data in the TradeChain database using the Temporal PageRank algorithm.
+        """
+
+        nodes = set()
+        G = nx.DiGraph()
+
+        for block in self.blocks:
+            if block.link_sequence_number == UNKNOWN_SEQ:
+                continue  # Don't consider half interactions
+
+            pubkey_requester = block.link_public_key
+            pubkey_responder = block.public_key
+
+            sequence_number_requester = block.link_sequence_number
+            sequence_number_responder = block.sequence_number
+
+            # In our market, we consider the amount of Bitcoin that have been transferred from A -> B.
+            # For now, we assume that the value from B -> A is of equal worth.
+
+            is_price_btc = block.transaction["tx"]["quantity_type"] == "BTC"
+            value_exchange = block.transaction["tx"]["quantity"] if is_price_btc else block.transaction["tx"]["price"]
+
+            G.add_edge((pubkey_requester, sequence_number_requester), (pubkey_requester, sequence_number_requester + 1),
+                       contribution=value_exchange)
+            G.add_edge((pubkey_requester, sequence_number_requester), (pubkey_responder, sequence_number_responder + 1),
+                       contribution=value_exchange)
+
+            G.add_edge((pubkey_responder, sequence_number_responder), (pubkey_responder, sequence_number_responder + 1),
+                       contribution=value_exchange)
+            G.add_edge((pubkey_responder, sequence_number_responder), (pubkey_requester, sequence_number_requester + 1),
+                       contribution=value_exchange)
+
+            nodes.add(pubkey_requester)
+            nodes.add(pubkey_responder)
+
+        personal_nodes = [node1 for node1 in G.nodes() if node1[0] == own_public_key]
+        number_of_nodes = len(personal_nodes)
+        if number_of_nodes == 0:
+            return {}
+        personalisation = {node_name: 1.0 / number_of_nodes if node_name in personal_nodes else 0
+                           for node_name in G.nodes()}
+
+        try:
+            result = nx.pagerank_scipy(G, personalization=personalisation, weight='contribution')
+        except nx.NetworkXException:
+            self._logger.info("Empty Temporal PageRank, returning empty scores")
+            return {}
+
+        sums = {}
+
+        for interaction in result.keys():
+            sums[interaction[0]] = sums.get(interaction[0], 0) + result[interaction]
+
+        return sums

doc/development/development_on_linux.rst

@@ -7,7 +7,7 @@ First, install the required dependencies by executing the following command in y
 
 .. code-block:: none
 
-    sudo apt-get install libav-tools libsodium18 libx11-6 python-apsw python-cherrypy3 python-cryptography python-decorator python-dnspython python-ecdsa python-feedparser python-jsonrpclib python-keyring python-keyrings.alt python-leveldb python-libtorrent python-matplotlib python-meliae python-m2crypto python-netifaces python-pbkdf2 python-pil python-protobuf python-pyasn1 python-pysocks python-requests python-twisted python2.7 vlc python-chardet python-configobj python-pyqt5 python-pyqt5.qtsvg
+    sudo apt-get install libav-tools libsodium18 libx11-6 python-apsw python-cherrypy3 python-cryptography python-decorator python-dnspython python-ecdsa python-feedparser python-jsonrpclib python-keyring python-keyrings.alt python-leveldb python-libtorrent python-matplotlib python-meliae python-m2crypto python-netifaces python-pbkdf2 python-pil python-protobuf python-pyasn1 python-pysocks python-requests python-scipy python-twisted python2.7 vlc python-chardet python-configobj python-pyqt5 python-pyqt5.qtsvg
 
 Next, download the latest .deb file from `here <https://jenkins.tribler.org/job/Build-Tribler_Ubuntu-64_devel/lastStableBuild/>`_.
 

doc/development/development_on_osx.rst

@@ -139,7 +139,7 @@ There are a bunch of other packages that can easily be installed using pip and b
     brew install homebrew/python/pillow gmp mpfr libmpc libsodium
     sudo easy_install pip
     pip install --user cython  # Needs to be installed first for meliae
-    pip install --user cherrypy cffi chardet configobj cryptography decorator dnspython ecdsa feedparser gmpy2 jsonrpclib idna keyring leveldb meliae netifaces numpy pbkdf2 pillow protobuf pyasn1 pysocks pycparser requests twisted service_identity
+    pip install --user cherrypy cffi chardet configobj cryptography decorator dnspython ecdsa feedparser gmpy2 jsonrpclib idna keyring leveldb meliae netifaces numpy pbkdf2 pillow protobuf pyasn1 pysocks pycparser requests scipy twisted service_identity
 
 If you encounter any error during the installation of Pillow, make sure that libjpeg and zlib are installed. They can be installed using: